fluent-plugin-elasticsearch 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90e3be14c80ad364758d6868ac2eeb3ff362d03c
-  data.tar.gz: 449226f062ac9c526d420e459fb25b731583d482
+  metadata.gz: 5313f204bb2cc56d58acf4587337bb9be25cabe5
+  data.tar.gz: bdf2ace0733655e50771305036256ec2a2486944
 SHA512:
-  metadata.gz: 0da741644cf316f00cb3aecf005933d465c076c4df8af080ec1a4d69bbd8d881de1298a5b734e1de2bf2e6cd36062579ae1e73a0e8684ec523dca3e7de89d84b
-  data.tar.gz: 1acd9c7b4e96059143dac1a4fc80eb8f170f74f934ee53304e112edaa01dadeccf4eb5fb44244ec4e16a55ef973a3534b60f61ea998249e892fecc675524886e
+  metadata.gz: 30f15c3bf195a5455c5d84f374934721a8550d25636a8e10d1410d6c6792e5f590f69b34129efcc1a9547bf0fefb68f703f82a61765fea4c603096a6d20a2568
+  data.tar.gz: 55ee1ed8366cd4c6624be2c216fa28428d42cbbecfcf6b8c1d87444761e91a55c83176376b91213b11f5e9edd6dd673d2845bb16ef03604688071a733a8617c4

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.DS_Store

data/.travis.yml

@@ -1,11 +1,9 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
   - 2.1
   - 2.2
-  - 2.3.0
+  - 2.3.1
 
 script: bundle exec rake test
 sudo: false

data/History.md

@@ -2,6 +2,12 @@
 
 ### Future
 
+### 1.6.0
+- add support for dot separated `target_index_key` and `target_type_key` (#175)
+- add `remove_keys_on_update` and `remove_keys_on_update_key` (#189)
+- fix support for fluentd v0.14 (#191)
+- remove support for elasticsearch v2 for now (#177)
+
 ### 1.5.0
 - add `routing_key` (#158)
 - add `time_key_exclude_timestamp` to exclude `@timestamp` (#161)

data/ISSUE_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
+
+**OR**
+
+Provide example config and message

data/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
-Pull Request Description HERE
+DESCRIPTION HERE
 
 (check all that apply)
 - [ ] tests added

data/README.md

@@ -35,6 +35,8 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [parent_key](#parent_key)
   + [routing_key](#routing_key)
   + [remove_keys](#remove_keys)
+  + [remove_keys_on_update](#remove_keys_on_update)
+  + [remove_keys_on_update_key](#remove_keys_on_update_key)
   + [write_operation](#write_operation)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
@@ -187,7 +189,7 @@ By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordina
 
 ### target_index_key
 
-Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms.
+Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
 
 If it is present in the record (and the value is non falsey) the value will be used as the index name to write to and then removed from the record before output; if it is not found then it will use logstash_format or index_name settings as configured.
 
@@ -220,7 +222,7 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 ### target_type_key
 
-Similar to `target_index_key` config, find the type name to write to in the record under this key. If key not found in record - fallback to `type_name` (default "fluentd").
+Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
 ### request_timeout
 
@@ -324,7 +326,30 @@ Similar to `parent_key` config, will add `_routing` into elasticsearch command i
 ```
 parent_key a_parent
 routing_key a_routing
-remove_keys a_parent, a_routing # a_parent and a_routing fileds wont be sent to elasticsearch
+remove_keys a_parent, a_routing # a_parent and a_routing fields won't be sent to elasticsearch
+```
+
+### remove_keys_on_update
+
+Remove keys on update will not update the configured keys in elasticsearch when a record is being updated.  
+This setting only has any effect if the write operation is update or upsert.
+
+If the write setting is upsert then these keys are only removed if the record is being
+updated, if the record does not exist (by id) then all of the keys are indexed.
+
+```
+remove_keys_on_update foo,bar
+```
+
+### remove_keys_on_update_key
+
+This setting allows `remove_keys_on_update` to be configured with a key in each record, in much the same way as `target_index_key` works.
+The configured key is removed before indexing in elasticsearch. If both `remove_keys_on_update` and `remove_keys_on_update_key` is
+present in the record then the keys in record are used, if the `remove_keys_on_update_key` is not present then the value of
+`remove_keys_on_update` is used as a fallback.
+
+```
+remove_keys_on_update_key keys_to_skip
 ```
 
 ### write_operation

data/Rakefile

@@ -5,6 +5,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.warning = false
 end
 
 task :default => :test

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.5.0'
+  s.version       = '1.6.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
@@ -16,9 +16,11 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ['lib']
 
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
+
   s.add_runtime_dependency 'fluentd', '>= 0.10.43'
   s.add_runtime_dependency 'excon', '>= 0'
-  s.add_runtime_dependency 'elasticsearch', '>= 0'
+  s.add_runtime_dependency 'elasticsearch', '< 1.1'
 
 
   s.add_development_dependency 'rake', '>= 0'

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -45,6 +45,8 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :client_key_pass, :string, :default => nil
   config_param :ca_file, :string, :default => nil
   config_param :remove_keys, :string, :default => nil
+  config_param :remove_keys_on_update, :string, :default => ""
+  config_param :remove_keys_on_update_key, :string, :default => nil
   config_param :flatten_hashes, :bool, :default => false
   config_param :flatten_hashes_separator, :string, :default => "_"
 
@@ -63,6 +65,18 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     if @remove_keys
       @remove_keys = @remove_keys.split(/\s*,\s*/)
     end
+
+    if @target_index_key && @target_index_key.is_a?(String)
+      @target_index_key = @target_index_key.split '.'
+    end
+
+    if @target_type_key && @target_type_key.is_a?(String)
+      @target_type_key = @target_type_key.split '.'
+    end
+
+    if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
+      @remove_keys_on_update = @remove_keys_on_update.split ','
+    end
   end
 
   def start
@@ -183,19 +197,41 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     when "update", "upsert"
       if meta.has_key?("_id")
         msgs << { "update" => meta }
-        msgs << { "doc" => record, "doc_as_upsert" => op == "upsert" }
+        msgs << update_body(record, op)
       end
     when "create"
       if meta.has_key?("_id")
         msgs << { "create" => meta }
         msgs << record
-      end        
+      end
     when "index"
       msgs << { "index" => meta }
       msgs << record
     end
   end
 
+  def update_body(record, op)
+    update = remove_keys(record)
+    body = { "doc" => update }
+    if  op == "upsert"
+      if update == record
+        body["doc_as_upsert"] = true
+      else
+        body["upsert"] = record
+      end
+    end
+    body
+  end
+
+  def remove_keys(record)
+    keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
+    record.delete(@remove_keys_on_update_key)
+    return record unless keys.any?
+    record = record.dup
+    keys.each { |key| record.delete(key) }
+    record
+  end
+
   def flatten_record(record, prefix=[])
     ret = {}
     if record.is_a? Hash
@@ -220,8 +256,9 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       end
 
       next unless record.is_a? Hash
-      if @target_index_key && record[@target_index_key]
-        target_index = record.delete @target_index_key
+      target_index_parent, target_index_child_key = get_parent_of(record, @target_index_key)
+      if target_index_parent && target_index_parent[target_index_child_key]
+        target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
         if record.has_key?("@timestamp")
           dt = record["@timestamp"]
@@ -238,21 +275,21 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       else
         target_index = @index_name
       end
-      
+
       # Change target_index to lower-case since Elasticsearch doesn't
       # allow upper-case characters in index names.
       target_index = target_index.downcase
-      
       if @include_tag_key
         record.merge!(@tag_key => tag)
       end
-      
-      if @target_type_key && record[@target_type_key]
-        target_type = record.delete @target_type_key
+
+      target_type_parent, target_type_child_key = get_parent_of(record, @target_type_key)
+      if target_type_parent && target_type_parent[target_type_child_key]
+        target_type = target_type_parent.delete(target_type_child_key)
       else
         target_type = @type_name
       end
-      
+
       meta = {"_index" => target_index, "_type" => target_type}
 
       @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' }
@@ -272,6 +309,15 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     bulk_message.clear
   end
 
+  # returns [parent, child_key] of child described by path array in record's tree
+  # returns [nil, child_key] if path doesnt exist in record
+  def get_parent_of(record, path)
+    return [nil, nil] unless path
+
+    parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
+    [parent_object, path[-1]]
+  end
+
   def send(data)
     retries = 0
     begin

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -21,7 +21,8 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     super
 
     # evaluate all configurations here
-    @dynamic_params = self.instance_variables.select { |var| is_valid_expand_param_type(var) }
+    @dynamic_params ||= []
+    @dynamic_params += self.instance_variables.select { |var| is_valid_expand_param_type(var) }
     @dynamic_config = Hash.new
     @dynamic_params.each { |var|
       value = expand_param(self.instance_variable_get(var), nil, nil, nil)
@@ -76,7 +77,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         if host_str.match(%r{^[^:]+(\:\d+)?$})
           {
             host:   host_str.split(':')[0],
-            port:   (host_str.split(':')[1] || @dynamic_config['port']).to_i,
+            port:   (host_str.split(':')[1] || @dynamic_config['port'] || @port).to_i,
             scheme: @dynamic_config['scheme']
           }
         else
@@ -127,7 +128,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       }
       # end eval all configs
 
-      if eval(dynamic_conf['logstash_format'])
+      if eval_or_val(dynamic_conf['logstash_format'])
         if record.has_key?("@timestamp")
           time = Time.parse record["@timestamp"]
         elsif record.has_key?(dynamic_conf['time_key'])
@@ -137,7 +138,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
           record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
         end
 
-        if eval(dynamic_conf['utc_index'])
+        if eval_or_val(dynamic_conf['utc_index'])
           target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
         else
           target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).strftime("#{dynamic_conf['logstash_dateformat']}")}"
@@ -145,11 +146,11 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       else
         target_index = dynamic_conf['index_name']
       end
-    
+
       # Change target_index to lower-case since Elasticsearch doesn't
       # allow upper-case characters in index names.
       target_index = target_index.downcase
-       
+
       if @include_tag_key
         record.merge!(dynamic_conf['tag_key'] => tag)
       end
@@ -198,6 +199,11 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     end
   end
 
+  def eval_or_val(var)
+    return var unless var.is_a?(String)
+    eval(var)
+  end
+
   def expand_param(param, tag, time, record)
     # check for '${ ... }'
     #   yes => `eval`

data/test/plugin/test_out_elasticsearch.rb

@@ -156,15 +156,15 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
-  
+
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
-	# Allthough index_name has upper-case characters,
-	# it should be set as lower-case when sent to elasticsearch.
+    # Allthough index_name has upper-case characters,
+    # it should be set as lower-case when sent to elasticsearch.
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
@@ -178,28 +178,28 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('local-override', index_cmds.first['index']['_index'])
     assert_nil(index_cmds[1]['@target_index'])
   end
-  
+
   def test_writes_to_target_index_key_logstash
     driver.configure("target_index_key @target_index\n")
     driver.configure("logstash_format true\n")
     time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('@target_index' => 'local-override'), time)
+    driver.emit(sample_record.merge('@target_index' => 'local-override'), time.to_i)
     driver.run
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
-  
+
    def test_writes_to_target_index_key_logstash_uppercase
     driver.configure("target_index_key @target_index\n")
     driver.configure("logstash_format true\n")
     time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('@target_index' => 'Local-Override'), time)
+    driver.emit(sample_record.merge('@target_index' => 'Local-Override'), time.to_i)
     driver.run
-	# Allthough @target_index has upper-case characters,
-	# it should be set as lower-case when sent to elasticsearch.
+    # Allthough @target_index has upper-case characters,
+    # it should be set as lower-case when sent to elasticsearch.
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
 
@@ -219,7 +219,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -263,6 +263,33 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
+  def test_writes_to_target_type_key_nested
+    driver.configure("target_type_key kubernetes.labels.log_type\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('kubernetes' => {
+      'labels' => {
+        'log_type' => 'local-override'
+      }
+    }))
+    driver.run
+    assert_equal('local-override', index_cmds.first['index']['_type'])
+    assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
+  end
+
+  def test_writes_to_target_type_key_fallack_to_default_nested
+    driver.configure("target_type_key kubernetes.labels.log_type\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('kubernetes' => {
+      'labels' => {
+        'other_labels' => 'test'
+      }
+    }))
+    driver.run
+    assert_equal('fluentd', index_cmds.first['index']['_type'])
+  end
+
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     stub_elastic_ping("http://192.168.33.50:9200")
@@ -373,7 +400,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "logstash-2015.05.31"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, dt.to_time)
+    driver.emit(sample_record, dt.to_time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -387,7 +414,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(index, index_cmds.first['index']['_index'])
   end
@@ -399,11 +426,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
-  
+
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")
@@ -411,10 +438,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
-	# Allthough logstash_prefix has upper-case characters,
-	# it should be set as lower-case when sent to elasticsearch.
+    # Allthough logstash_prefix has upper-case characters,
+    # it should be set as lower-case when sent to elasticsearch.
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
@@ -425,7 +452,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -438,7 +465,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -724,7 +751,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_update_should_write_update_op_and_doc_as_upsert_is_false
-    driver.configure("write_operation update 
+    driver.configure("write_operation update
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
@@ -732,10 +759,23 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert(index_cmds[0].has_key?("update"))
     assert(!index_cmds[1]["doc_as_upsert"])
+    assert(!index_cmds[1]["upsert"])
+  end
+
+  def test_update_should_remove_keys_from_doc_when_keys_are_skipped
+    driver.configure("write_operation update
+                      id_key request_id
+                      remove_keys_on_update parent_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[1]["doc"])
+    assert(!index_cmds[1]["doc"]["parent_id"])
   end
 
   def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
-    driver.configure("write_operation upsert 
+    driver.configure("write_operation upsert
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
@@ -743,10 +783,111 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert(index_cmds[0].has_key?("update"))
     assert(index_cmds[1]["doc_as_upsert"])
+    assert(!index_cmds[1]["upsert"])
+  end
+
+  def test_upsert_should_write_update_op_upsert_and_doc_when_keys_are_skipped
+    driver.configure("write_operation upsert
+                      id_key request_id
+                      remove_keys_on_update parent_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[0].has_key?("update"))
+    assert(!index_cmds[1]["doc_as_upsert"])
+    assert(index_cmds[1]["upsert"])
+    assert(index_cmds[1]["doc"])
+  end
+
+  def test_upsert_should_remove_keys_from_doc_when_keys_are_skipped
+    driver.configure("write_operation upsert
+                      id_key request_id
+                      remove_keys_on_update parent_id")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
+    assert(!index_cmds[1]["doc"]["parent_id"])
+    assert(index_cmds[1]["upsert"]["parent_id"])
+  end
+
+  def test_upsert_should_remove_multiple_keys_when_keys_are_skipped
+    driver.configure("write_operation upsert
+                      id_key id
+                      remove_keys_on_update foo,baz")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
+    driver.run
+    assert(
+      index_cmds[1]["doc"] == {
+        "id" => 1,
+        "zip" => "zam",
+      }
+    )
+    assert(
+      index_cmds[1]["upsert"] == {
+        "id" => 1,
+        "foo" => "bar",
+        "baz" => "quix",
+        "zip" => "zam",
+      }
+    )
+  end
+
+  def test_upsert_should_remove_keys_from_when_the_keys_are_in_the_record
+    driver.configure("write_operation upsert
+                      id_key id
+                      remove_keys_on_update_key keys_to_skip")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
+    driver.run
+    assert(
+      index_cmds[1]["doc"] == {
+        "id" => 1,
+        "foo" => "bar",
+      }
+    )
+    assert(
+      index_cmds[1]["upsert"] == {
+        "id" => 1,
+        "foo" => "bar",
+        "baz" => "quix",
+      }
+    )
+  end
+
+  def test_upsert_should_remove_keys_from_key_on_record_has_higher_presedence_than_config
+    driver.configure("write_operation upsert
+                      id_key id
+                      remove_keys_on_update foo,bar
+                      remove_keys_on_update_key keys_to_skip")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
+    driver.run
+    assert(
+      index_cmds[1]["doc"] == {
+        "id" => 1,
+        # we only expect baz to be stripped here, if the config was more important
+        # foo would be stripped too.
+        "foo" => "bar",
+      }
+    )
+    assert(
+      index_cmds[1]["upsert"] == {
+        "id" => 1,
+        "foo" => "bar",
+        "baz" => "quix",
+      }
+    )
   end
 
   def test_create_should_write_create_op
-    driver.configure("write_operation create 
+    driver.configure("write_operation create
                       id_key request_id")
     stub_elastic_ping
     stub_elastic

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -56,7 +56,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     instance = driver('test', config).instance
 
     assert_equal 'logs.google.com', instance.host
-    assert_equal "777", instance.port
+    assert_equal 777, instance.port
     assert_equal 'https', instance.scheme
     assert_equal '/es/', instance.path
     assert_equal 'john', instance.user
@@ -154,7 +154,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
-  
+
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
@@ -247,7 +247,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -259,7 +259,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(utc_index, index_cmds.first['index']['_index'])
   end
@@ -271,11 +271,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
-  
+
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")
@@ -283,7 +283,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -295,7 +295,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -308,7 +308,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, time.to_i)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
@@ -546,7 +546,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_update_should_write_update_op_and_doc_as_upsert_is_false
-    driver.configure("write_operation update 
+    driver.configure("write_operation update
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
@@ -557,7 +557,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
-    driver.configure("write_operation upsert 
+    driver.configure("write_operation upsert
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
@@ -568,7 +568,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_create_should_write_create_op
-    driver.configure("write_operation create 
+    driver.configure("write_operation create
                       id_key request_id")
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-11 00:00:00.000000000 Z
+date: 2016-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -43,16 +43,16 @@ dependencies:
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +123,7 @@ files:
 - ".travis.yml"
 - Gemfile
 - History.md
+- ISSUE_TEMPLATE.md
 - LICENSE.txt
 - PULL_REQUEST_TEMPLATE.md
 - README.md
@@ -145,7 +146,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="