fluent-plugin-elasticsearch 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 129193e2eba365974ee13d84c4cdda02c76f2e78
-  data.tar.gz: 5f8d50deb55473586b8400fb802cac87faab9370
+  metadata.gz: 1e5c36c889794052ff9bab9d9b6e69aa8d937eed
+  data.tar.gz: a719c42134c8d53969abe517a47f47a6936d092d
 SHA512:
-  metadata.gz: 0680f132de350ab43743328b68b785a663ced2bafadc010d6f16c185d852b40321e7326487c24e095c67af53603d0518d7868bc4ddb21dffe9b240e6365ff42a
-  data.tar.gz: 32640f604a3f5b2549b5a4eceb0f7e4a61a590b0027c2636f810a6de077ac60dd88a338907e1dfd7866205c7e54bf18fd35dba14245db146dd5eed4a4a30483c
+  metadata.gz: e75bdccd2859b063eb1fc48fdce8394e43648ff05b4d25214b66b85111482c1d8d671e24d3a5376d37f0f1093fbefb2866afc5e9882728df2b09c2147cf16a90
+  data.tar.gz: 3393dce7f111c75ce16b8b4af7fff51064870cb1033bf446701180cf0c84458ffc3a8ea3f6a58503eff46d1d7e414e588073497fd5e6a20becc036971ddb40d8

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-ci
+

data/.travis.yml

@@ -7,3 +7,4 @@ rvm:
   - 2.2
 
 script: bundle exec rake test
+sudo: false

data/Gemfile

@@ -3,4 +3,5 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
 gemspec
 
+gem 'simplecov', require: false
 gem 'coveralls', require: false

data/History.md

@@ -1,6 +1,7 @@
 ## Changelog
 
-### Future
+### 1.3.0
+- add `write_operation`
 
 ### 1.2.1
 - fix `resurrect_after` in out_elasticsearch_dynamic

data/README.md

@@ -1,10 +1,11 @@
 # Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
-[![Dependency Status](https://gemnasium.com/uken/guard-sidekiq.png)](https://gemnasium.com/uken/fluent-plugin-elasticsearch)
 [![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
 [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
+[![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-elasticsearch/badge/pr)](http://issuestats.com/github/uken/fluent-plugin-elasticsearch)
+[![Issue Stats](http://issuestats.com/github/uken/fluent-plugin-elasticsearch/badge/issue)](http://issuestats.com/github/uken/fluent-plugin-elasticsearch)
 
 Send your logs to ElasticSearch (and search them with Kibana maybe?)
 
@@ -27,6 +28,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [resurrect_after](#resurrect_after)
   + [include_tag_key, tag_key](#include_tag_key-tag_key)
   + [id_key](#id_key)
+  + [write_operation](#write_operation)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Buffered output options](#buffered-output-options)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
@@ -164,21 +166,21 @@ request_timeout 15s # defaults to 5s
 
 ### reload_connections
 
+You can tune how the elasticsearch-transport host reloading feature works. By default it will reload the host list from the server every 10,000th request to spread the load. This can be an issue if your ElasticSearch cluster is behind a Reverse Proxy, as Fluentd process may not have direct network access to the ElasticSearch nodes.
+
 ```
 reload_connections false # defaults to true
 ```
 
 ### reload_on_failure
 
-You can tune how the elasticsearch-transport host reloading feature works. By default it will reload the host list from the server every 10,000th request to spread the load. This can be an issue if your ElasticSearch cluster is behind a Reverse Proxy, as Fluentd process may not have direct network access to the ElasticSearch nodes.
+Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the
+request, this can be useful to quickly remove a dead node from the list of addresses.
 
 ```
 reload_on_failure true # defaults to false
 ```
 
-Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the
-request, this can be useful to quickly remove a dead node from the list of addresses.
-
 ### resurrect_after
 
 You can set in the elasticsearch-transport how often dead connections from the elasticsearch-transport's pool will be resurrected.
@@ -225,6 +227,19 @@ This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will tr
 { "name": "Johnny", "request_id": "87d89af7daffad6" }
 ```
 
+### write_operation
+
+The write_operation can be any of: 
+
+| Operation | Description          |
+| ------------- | ----------- |
+| index (default)      | new data is added while existing data (based on its id) is replaced (reindexed).|
+| create      | adds new data - if the data already exists (based on its id), the op is skipped.|
+| update      | updates existing data (based on its id). If no data is found, the op is skipped.|
+| upsert      | known as merge or insert if the data does not exist, updates if the data exists (based on its id).|
+
+**Please note, id is required in create, update, and upsert scenario. Without id, the message will be dropped.**
+
 ### Client/host certificate options
 
 Need to verify ElasticSearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/Rakefile

@@ -1,11 +1,10 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
 require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
+  test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
 end
 
 task :default => :test
-

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.2.1'
+  s.version       = '1.3.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
@@ -21,8 +21,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', '>= 0'
 
 
-  s.add_development_dependency 'rake', '~> 0'
+  s.add_development_dependency 'rake', '>= 0'
   s.add_development_dependency 'webmock', '~> 1'
   s.add_development_dependency 'test-unit', '~> 3.1.0'
-  s.add_development_dependency 'minitest', '~> 5.7.0'
+  s.add_development_dependency 'minitest', '~> 5.8'
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -23,6 +23,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :type_name, :string, :default => "fluentd"
   config_param :index_name, :string, :default => "fluentd"
   config_param :id_key, :string, :default => nil
+  config_param :write_operation, :string, :default => "index"
   config_param :parent_key, :string, :default => nil
   config_param :request_timeout, :time, :default => 5
   config_param :reload_connections, :bool, :default => true
@@ -127,6 +128,24 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     super
   end
 
+  def append_record_to_messages(op, meta, record, msgs)
+    case op
+    when "update", "upsert"
+      if meta.has_key?("_id")
+        msgs << { "update" => meta }
+        msgs << { "doc" => record, "doc_as_upsert" => op == "upsert" }
+      end
+    when "create"
+      if meta.has_key?("_id")
+        msgs << { "create" => meta }
+        msgs << record
+      end        
+    when "index"
+      msgs << { "index" => meta }
+      msgs << record
+    end
+  end
+
   def write(chunk)
     bulk_message = []
 
@@ -154,17 +173,16 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
         record.merge!(@tag_key => tag)
       end
 
-      meta = { "index" => {"_index" => target_index, "_type" => type_name} }
+      meta = {"_index" => target_index, "_type" => type_name}
       if @id_key && record[@id_key]
-        meta['index']['_id'] = record[@id_key]
+        meta['_id'] = record[@id_key]
       end
 
       if @parent_key && record[@parent_key]
-        meta['index']['_parent'] = record[@parent_key]
+        meta['_parent'] = record[@parent_key]
       end
 
-      bulk_message << meta
-      bulk_message << record
+      append_record_to_messages(@write_operation, meta, record, bulk_message)
     end
 
     send(bulk_message) unless bulk_message.empty?

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -149,13 +149,13 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         record.merge!(dynamic_conf['tag_key'] => tag)
       end
 
-      meta = { "index" => {"_index" => target_index, "_type" => dynamic_conf['type_name']} }
+      meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
       if dynamic_conf['id_key'] && record[dynamic_conf['id_key']]
-        meta['index']['_id'] = record[dynamic_conf['id_key']]
+        meta['_id'] = record[dynamic_conf['id_key']]
       end
 
       if dynamic_conf['parent_key'] && record[dynamic_conf['parent_key']]
-        meta['index']['_parent'] = record[dynamic_conf['parent_key']]
+        meta['_parent'] = record[dynamic_conf['parent_key']]
       end
 
       if dynamic_conf['hosts']
@@ -164,9 +164,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
       end
 
-      bulk_message[host] << meta
-      bulk_message[host] << record
-
+      append_record_to_messages(dynamic_conf["write_operation"], meta, record, bulk_message[host])
     end
 
     bulk_message.each do | hKey, array |

data/test/helper.rb

@@ -1 +1,24 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter do |src|
+    !(src.filename =~ /^#{SimpleCov.root}\/lib/)
+  end
+end
+
+require 'coveralls'
+Coveralls.wear!
+
+# needs to be after simplecov but before test/unit, because fluentd sets default
+# encoding to ASCII-8BIT, but coverall might load git data which could contain a
+# UTF-8 character
+at_exit do
+  Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
+  Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
+end
+
+require 'test/unit'
+require 'fluent/test'
 require 'minitest/pride'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/plugin/test_out_elasticsearch.rb

@@ -1,23 +1,12 @@
-require 'test/unit'
-
-require 'fluent/test'
-require 'fluent/plugin/out_elasticsearch'
-
-require 'webmock/test_unit'
-require 'date'
-
-$:.push File.expand_path("../..", __FILE__)
-$:.push File.dirname(__FILE__)
-
 require 'helper'
-
-WebMock.disable_net_connect!
+require 'date'
 
 class ElasticsearchOutput < Test::Unit::TestCase
   attr_accessor :index_cmds, :index_command_counts
 
   def setup
     Fluent::Test.setup
+    require 'fluent/plugin/out_elasticsearch'
     @driver = nil
   end
 
@@ -449,4 +438,64 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
     assert_equal(connection_resets, 3)
   end
+
+  def test_update_should_not_write_if_theres_no_id
+    driver.configure("write_operation update\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_upsert_should_not_write_if_theres_no_id
+    driver.configure("write_operation upsert\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_create_should_not_write_if_theres_no_id
+    driver.configure("write_operation create\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_update_should_write_update_op_and_doc_as_upsert_is_false
+    driver.configure("write_operation update 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("update"))
+    assert(!index_cmds[1]["doc_as_upsert"])
+  end
+
+  def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
+    driver.configure("write_operation upsert 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("update"))
+    assert(index_cmds[1]["doc_as_upsert"])
+  end
+
+  def test_create_should_write_create_op
+    driver.configure("write_operation create 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("create"))
+  end
+
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -1,23 +1,12 @@
-require 'test/unit'
-
-require 'fluent/test'
-require 'fluent/plugin/out_elasticsearch_dynamic'
-
-require 'webmock/test_unit'
-require 'date'
-
-$:.push File.expand_path("../..", __FILE__)
-$:.push File.dirname(__FILE__)
-
 require 'helper'
-
-WebMock.disable_net_connect!
+require 'date'
 
 class ElasticsearchOutputDynamic < Test::Unit::TestCase
   attr_accessor :index_cmds, :index_command_counts
 
   def setup
     Fluent::Test.setup
+    require 'fluent/plugin/out_elasticsearch_dynamic'
     @driver = nil
   end
 
@@ -449,4 +438,63 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
     assert_equal(connection_resets, 3)
   end
+
+  def test_update_should_not_write_if_theres_no_id
+    driver.configure("write_operation update\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_upsert_should_not_write_if_theres_no_id
+    driver.configure("write_operation upsert\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_create_should_not_write_if_theres_no_id
+    driver.configure("write_operation create\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds)
+  end
+
+  def test_update_should_write_update_op_and_doc_as_upsert_is_false
+    driver.configure("write_operation update 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("update"))
+    assert(!index_cmds[1]["doc_as_upsert"])
+  end
+
+  def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
+    driver.configure("write_operation upsert 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("update"))
+    assert(index_cmds[1]["doc_as_upsert"])
+  end
+
+  def test_create_should_write_create_op
+    driver.configure("write_operation create 
+                      id_key request_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("create"))
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-03 00:00:00.000000000 Z
+date: 2016-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -57,14 +57,14 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -101,14 +101,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.7.0
+        version: '5.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.7.0
+        version: '5.8'
 description: ElasticSearch output plugin for Fluent event collector
 email:
 - pitr.vern@gmail.com
@@ -117,6 +117,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".editorconfig"
 - ".gitignore"
 - ".travis.yml"