fluent-plugin-elasticsearch 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89a28c84005c362158da08a71ea7c10036df254d
-  data.tar.gz: 87a5e6283b6e3bd057ec949ee95316460bc8f6da
+  metadata.gz: 90e3be14c80ad364758d6868ac2eeb3ff362d03c
+  data.tar.gz: 449226f062ac9c526d420e459fb25b731583d482
 SHA512:
-  metadata.gz: fde384e1b97967886d1446785e1cdb5eedb1bf7f98ab7841d321d34886dc84ee95f255d8d86b289c03b40e469528efce6a288b63072a6fd9f40f5451b82bd043
-  data.tar.gz: 35d51c4cc3508cf12e6c4971db4ab4c947c4688cc4f021a33c4d0bf1f325a03a013b1b1b30195be8a64296636997e757a518fdd5516234741824cec20151197b
+  metadata.gz: 0da741644cf316f00cb3aecf005933d465c076c4df8af080ec1a4d69bbd8d881de1298a5b734e1de2bf2e6cd36062579ae1e73a0e8684ec523dca3e7de89d84b
+  data.tar.gz: 1acd9c7b4e96059143dac1a4fc80eb8f170f74f934ee53304e112edaa01dadeccf4eb5fb44244ec4e16a55ef973a3534b60f61ea998249e892fecc675524886e

data/History.md

@@ -1,7 +1,15 @@
-## Changelog
+## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### Future
 
+### 1.5.0
+- add `routing_key` (#158)
+- add `time_key_exclude_timestamp` to exclude `@timestamp` (#161)
+- convert index names to lowercase (#163)
+- add `remove_keys` (#164)
+- add `flatten_hashes` (#168)
+- add `target_type_key` (#169)
+
 ### 1.4.0
 - add `target_index_key` to specify target index (#153)
 - add `time_key_format` for faster time format parsing (#154)

data/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,10 @@
+Pull Request Description HERE
+
+(check all that apply)
+- [ ] tests added
+- [ ] tests passing
+- [ ] README updated (if needed)
+- [ ] README Table of Contents updated (if needed)
+- [ ] History.md and `version` in gemspec are untouched
+- [ ] backward compatible
+- [ ] feature works in `elasticsearch_dynamic` (not required but recommended)

data/README.md

@@ -22,18 +22,24 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [logstash_dateformat](#logstash_dateformat)
   + [time_key_format](#time_key_format)
   + [time_key](#time_key)
+  + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
   + [utc_index](#utc_index)
   + [target_index_key](#target_index_key)
+  + [target_type_key](#target_type_key)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
   + [resurrect_after](#resurrect_after)
   + [include_tag_key, tag_key](#include_tag_key-tag_key)
   + [id_key](#id_key)
+  + [parent_key](#parent_key)
+  + [routing_key](#routing_key)
+  + [remove_keys](#remove_keys)
   + [write_operation](#write_operation)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
+  + [Hash flattening](#hash-flattening)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
 * [Contact](#contact)
@@ -117,7 +123,7 @@ logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
 
 ### time_key_format
 
-The format of the time stamp field (`@timestamp` or what you specify with [time_key][#time_key]). This parameter only has an effect when [logstash_format][#logstash_format] is true as it only affects the name of the index we write to. Please see [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime) for information about the value of this format.
+The format of the time stamp field (`@timestamp` or what you specify with [time_key](#time_key)). This parameter only has an effect when [logstash_format](#logstash_format) is true as it only affects the name of the index we write to. Please see [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime) for information about the value of this format.
 
 Setting this to a known format can vastly improve your log ingestion speed if all most of your logs are in the same format. If there is an error parsing this format the timestamp will default to the ingestion time. If you are on Ruby 2.0 or later you can get a further performance improvment by installing the "strptime" gem: `fluent-gem install strptime`.
 
@@ -161,6 +167,16 @@ The output will be
 }
 ```
 
+See `time_key_exclude_timestamp` to avoid adding `@timestamp`.
+
+### time_key_exclude_timestamp
+
+```
+time_key_exclude_timestamp false
+```
+
+By default, setting `time_key` will copy the value to an additional field `@timestamp`. When setting `time_key_exclude_timestamp true`, no additional field will be added.
+
 ### utc_index
 
 ```
@@ -202,6 +218,10 @@ The output would be
 
 and this record will be written to the specified index (`logstash-2014.12.19`) rather than `fallback`.
 
+### target_type_key
+
+Similar to `target_index_key` config, find the type name to write to in the record under this key. If key not found in record - fallback to `type_name` (default "fluentd").
+
 ### request_timeout
 
 You can specify HTTP request timeout.
@@ -275,6 +295,38 @@ This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will tr
 { "name": "Johnny", "request_id": "87d89af7daffad6" }
 ```
 
+### parent_key
+
+```
+parent_key a_parent # use "a_parent" field value to set _parent in elasticsearch command
+```
+
+If your input is
+```
+{ "name": "Johnny", "a_parent": "my_parent" }
+```
+
+ElasticSearch command would be
+
+```
+{ "index" : { "_index" : "****", "_type" : "****", "_id" : "****", "_parent" : "my_parent" } }
+{ "name": "Johnny", "a_parent": "my_parent" }
+```
+
+if `parent_key` is not configed or the `parent_key` is absent in input record, nothing will happen.
+
+### routing_key
+
+Similar to `parent_key` config, will add `_routing` into elasticsearch command if `routing_key` is set and the field does exist in input event.
+
+### remove_keys
+
+```
+parent_key a_parent
+routing_key a_routing
+remove_keys a_parent, a_routing # a_parent and a_routing fileds wont be sent to elasticsearch
+```
+
 ### write_operation
 
 The write_operation can be any of:
@@ -318,7 +370,26 @@ retry_wait 1.0
 num_threads 1
 ```
 
-The value for option `buffer_chunk_limit` should not exceed value `http.max_content_length` in your Elasticsearch setup (by default it is 104857600 bytes).
+The value for option `buffer_chunk_limit` should not exceed value `http.max_content_length` in your Elasticsearch setup (by default it is 100mb).
+
+### Hash flattening
+
+Elasticsearch will complain if you send object and concrete values to the same field. For example, you might have logs that look this, from different places:
+
+{"people" => 100}
+{"people" => {"some" => "thing"}}
+
+The second log line will be rejected by the Elasticsearch parser because objects and concrete values can't live in the same field. To combat this, you can enable hash flattening.
+
+```
+flatten_hashes true
+flatten_hashes_separator _
+```
+
+This will produce elasticsearch output that looks like this:
+{"people_some" => "thing"}
+
+Note that the flattener does not deal with arrays at this time.
 
 ### Not seeing a config you need?
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.4.0'
+  s.version       = '1.5.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -21,6 +21,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :scheme, :string, :default => 'http'
   config_param :hosts, :string, :default => nil
   config_param :target_index_key, :string, :default => nil
+  config_param :target_type_key, :string, :default => nil
   config_param :time_key_format, :string, :default => nil
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
@@ -31,16 +32,21 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :id_key, :string, :default => nil
   config_param :write_operation, :string, :default => "index"
   config_param :parent_key, :string, :default => nil
+  config_param :routing_key, :string, :default => nil
   config_param :request_timeout, :time, :default => 5
   config_param :reload_connections, :bool, :default => true
   config_param :reload_on_failure, :bool, :default => false
   config_param :resurrect_after, :time, :default => 60
   config_param :time_key, :string, :default => nil
+  config_param :time_key_exclude_timestamp, :bool, :default => false
   config_param :ssl_verify , :bool, :default => true
   config_param :client_key, :string, :default => nil
   config_param :client_cert, :string, :default => nil
   config_param :client_key_pass, :string, :default => nil
   config_param :ca_file, :string, :default => nil
+  config_param :remove_keys, :string, :default => nil
+  config_param :flatten_hashes, :bool, :default => false
+  config_param :flatten_hashes_separator, :string, :default => "_"
 
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
@@ -53,6 +59,10 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   def configure(conf)
     super
     @time_parser = TimeParser.new(@time_key_format, @router)
+
+    if @remove_keys
+      @remove_keys = @remove_keys.split(/\s*,\s*/)
+    end
   end
 
   def start
@@ -186,10 +196,29 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     end
   end
 
+  def flatten_record(record, prefix=[])
+    ret = {}
+    if record.is_a? Hash
+      record.each { |key, value|
+        ret.merge! flatten_record(value, prefix + [key.to_s])
+      }
+    elsif record.is_a? Array
+      # Don't mess with arrays, leave them unprocessed
+      ret.merge!({prefix.join(@flatten_hashes_separator) => record})
+    else
+      return {prefix.join(@flatten_hashes_separator) => record}
+    end
+    ret
+  end
+
   def write(chunk)
     bulk_message = []
 
     chunk.msgpack_each do |tag, time, record|
+      if @flatten_hashes
+        record = flatten_record(record)
+      end
+
       next unless record.is_a? Hash
       if @target_index_key && record[@target_index_key]
         target_index = record.delete @target_index_key
@@ -199,7 +228,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
           dt = @time_parser.parse(record["@timestamp"], time)
         elsif record.has_key?(@time_key)
           dt = @time_parser.parse(record[@time_key], time)
-          record['@timestamp'] = record[@time_key]
+          record['@timestamp'] = record[@time_key] unless time_key_exclude_timestamp
         else
           dt = Time.at(time).to_datetime
           record.merge!({"@timestamp" => dt.to_s})
@@ -209,18 +238,31 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       else
         target_index = @index_name
       end
-
+      
+      # Change target_index to lower-case since Elasticsearch doesn't
+      # allow upper-case characters in index names.
+      target_index = target_index.downcase
+      
       if @include_tag_key
         record.merge!(@tag_key => tag)
       end
+      
+      if @target_type_key && record[@target_type_key]
+        target_type = record.delete @target_type_key
+      else
+        target_type = @type_name
+      end
+      
+      meta = {"_index" => target_index, "_type" => target_type}
 
-      meta = {"_index" => target_index, "_type" => type_name}
-      if @id_key && record[@id_key]
-        meta['_id'] = record[@id_key]
+      @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' }
+      @meta_config_map.each_pair do |config_name, meta_key|
+        record_key = self.instance_variable_get("@#{config_name}")
+        meta[meta_key] = record[record_key] if record_key && record[record_key]
       end
 
-      if @parent_key && record[@parent_key]
-        meta['_parent'] = record[@parent_key]
+      if @remove_keys
+        @remove_keys.each { |key| record.delete(key) }
       end
 
       append_record_to_messages(@write_operation, meta, record, bulk_message)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -11,6 +11,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   config_param :port, :string, :default => "9200"
   config_param :logstash_format, :string, :default => "false"
   config_param :utc_index, :string, :default => "true"
+  config_param :time_key_exclude_timestamp, :bool, :default => false
   config_param :reload_connections, :string, :default => "true"
   config_param :reload_on_failure, :string, :default => "false"
   config_param :resurrect_after, :string, :default => "60"
@@ -131,7 +132,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
           time = Time.parse record["@timestamp"]
         elsif record.has_key?(dynamic_conf['time_key'])
           time = Time.parse record[dynamic_conf['time_key']]
-          record['@timestamp'] = record[dynamic_conf['time_key']]
+          record['@timestamp'] = record[dynamic_conf['time_key']] unless time_key_exclude_timestamp
         else
           record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
         end
@@ -144,18 +145,22 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       else
         target_index = dynamic_conf['index_name']
       end
-
+    
+      # Change target_index to lower-case since Elasticsearch doesn't
+      # allow upper-case characters in index names.
+      target_index = target_index.downcase
+       
       if @include_tag_key
         record.merge!(dynamic_conf['tag_key'] => tag)
       end
 
       meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
-      if dynamic_conf['id_key'] && record[dynamic_conf['id_key']]
-        meta['_id'] = record[dynamic_conf['id_key']]
-      end
 
-      if dynamic_conf['parent_key'] && record[dynamic_conf['parent_key']]
-        meta['_parent'] = record[dynamic_conf['parent_key']]
+      @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' }
+      @meta_config_map.each_pair do |config_name, meta_key|
+        if dynamic_conf[config_name] && record[dynamic_conf[config_name]]
+          meta[meta_key] = record[dynamic_conf[config_name]]
+        end
       end
 
       if dynamic_conf['hosts']
@@ -164,6 +169,10 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
       end
 
+      if @remove_keys
+        @remove_keys.each { |key| record.delete(key) }
+      end
+
       append_record_to_messages(dynamic_conf["write_operation"], meta, record, bulk_message[host])
     end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -17,7 +17,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def sample_record
-    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent'}
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
 
   def stub_elastic_ping(url="http://localhost:9200")
@@ -156,6 +156,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
+  
+  def test_writes_to_speficied_index_uppercase
+    driver.configure("index_name MyIndex\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+	# Allthough index_name has upper-case characters,
+	# it should be set as lower-case when sent to elasticsearch.
+    assert_equal('myindex', index_cmds.first['index']['_index'])
+  end
 
   def test_writes_to_target_index_key
     driver.configure("target_index_key @target_index\n")
@@ -167,7 +178,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('local-override', index_cmds.first['index']['_index'])
     assert_nil(index_cmds[1]['@target_index'])
   end
-
+  
   def test_writes_to_target_index_key_logstash
     driver.configure("target_index_key @target_index\n")
     driver.configure("logstash_format true\n")
@@ -178,6 +189,19 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
+  
+   def test_writes_to_target_index_key_logstash_uppercase
+    driver.configure("target_index_key @target_index\n")
+    driver.configure("logstash_format true\n")
+    time = Time.parse Date.today.to_s
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('@target_index' => 'Local-Override'), time)
+    driver.run
+	# Allthough @target_index has upper-case characters,
+	# it should be set as lower-case when sent to elasticsearch.
+    assert_equal('local-override', index_cmds.first['index']['_index'])
+  end
 
   def test_writes_to_target_index_key_fallack
     driver.configure("target_index_key @target_index\n")
@@ -209,6 +233,36 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
+  def test_writes_to_target_type_key
+    driver.configure("target_type_key @target_type\n")
+    stub_elastic_ping
+    stub_elastic
+    record = sample_record.clone
+    driver.emit(sample_record.merge('@target_type' => 'local-override'))
+    driver.run
+    assert_equal('local-override', index_cmds.first['index']['_type'])
+    assert_nil(index_cmds[1]['@target_type'])
+  end
+
+  def test_writes_to_target_type_key_fallack_to_default
+    driver.configure("target_type_key @target_type\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('fluentd', index_cmds.first['index']['_type'])
+  end
+
+  def test_writes_to_target_type_key_fallack_to_type_name
+    driver.configure("target_type_key @target_type\n")
+    driver.configure("type_name mytype\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('mytype', index_cmds.first['index']['_type'])
+  end
+
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     stub_elastic_ping("http://192.168.33.50:9200")
@@ -258,6 +312,40 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(2000, total)
   end
 
+  def test_nested_record_with_flattening_on
+    driver.configure("flatten_hashes true
+                      flatten_hashes_separator |")
+
+    original_hash =  {"foo" => {"bar" => "baz"}, "people" => [
+      {"age" => "25", "height" => "1ft"},
+      {"age" => "30", "height" => "2ft"}
+    ]}
+
+    expected_output = {"foo|bar"=>"baz", "people" => [
+      {"age" => "25", "height" => "1ft"},
+      {"age" => "30", "height" => "2ft"}
+    ]}
+
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(original_hash)
+    driver.run
+    assert_equal expected_output, index_cmds[1]
+  end
+
+  def test_nested_record_with_flattening_off
+    # flattening off by default
+
+    original_hash =  {"foo" => {"bar" => "baz"}}
+    expected_output = {"foo" => {"bar" => "baz"}}
+
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(original_hash)
+    driver.run
+    assert_equal expected_output, index_cmds[1]
+  end
+
   def test_makes_bulk_request
     stub_elastic_ping
     stub_elastic
@@ -315,6 +403,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
+  
+  def test_writes_to_logstash_index_with_specified_prefix_uppercase
+    driver.configure("logstash_format true
+                      logstash_prefix MyPrefix")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+	# Allthough logstash_prefix has upper-case characters,
+	# it should be set as lower-case when sent to elasticsearch.
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
 
     def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
@@ -383,6 +485,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_exclude_timekey
+    driver.configure("logstash_format true
+                      time_key vtm
+                      time_key_exclude_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
+  end
 
   def test_uses_custom_time_key_format
     driver.configure("logstash_format true
@@ -502,6 +615,52 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert(!index_cmds[0]['index'].has_key?('_parent'))
   end
 
+  def test_adds_routing_key_when_configured
+    driver.configure("routing_key routing_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_routing'], 'routing')
+  end
+
+  def test_doesnt_add_routing_key_if_missing_when_configured
+    driver.configure("routing_key another_routing_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_routing'))
+  end
+
+  def test_adds_routing_key_when_not_configured
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_routing'))
+  end
+
+  def test_remove_one_key
+    driver.configure("remove_keys key1\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    driver.run
+    assert(!index_cmds[1].has_key?('key1'))
+    assert(index_cmds[1].has_key?('key2'))
+  end
+
+  def test_remove_multi_keys
+    driver.configure("remove_keys key1, key2\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    driver.run
+    assert(!index_cmds[1].has_key?('key1'))
+    assert(!index_cmds[1].has_key?('key2'))
+  end
+
   def test_request_error
     stub_elastic_ping
     stub_elastic_unavailable

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -15,7 +15,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def sample_record
-    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent'}
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
 
   def stub_elastic_ping(url="http://localhost:9200")
@@ -154,6 +154,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
+  
+  def test_writes_to_speficied_index_uppercase
+    driver.configure("index_name MyIndex\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('myindex', index_cmds.first['index']['_index'])
+  end
 
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
@@ -266,6 +275,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
+  
+  def test_writes_to_logstash_index_with_specified_prefix_uppercase
+    driver.configure("logstash_format true
+                      logstash_prefix MyPrefix")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
 
     def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
@@ -334,6 +355,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_exclude_timestamp
+    driver.configure("logstash_format true
+                      time_key vtm
+                      time_key_exclude_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
+  end
+
   def test_doesnt_add_tag_key_by_default
     stub_elastic_ping
     stub_elastic
@@ -404,6 +437,52 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert(!index_cmds[0]['index'].has_key?('_parent'))
   end
 
+  def test_adds_routing_key_when_configured
+    driver.configure("routing_key routing_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_routing'], 'routing')
+  end
+
+  def test_doesnt_add_routing_key_if_missing_when_configured
+    driver.configure("routing_key another_routing_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_routing'))
+  end
+
+  def test_adds_routing_key_when_not_configured
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_routing'))
+  end
+
+  def test_remove_one_key
+    driver.configure("remove_keys key1\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    driver.run
+    assert(!index_cmds[1].has_key?('key1'))
+    assert(index_cmds[1].has_key?('key2'))
+  end
+
+  def test_remove_multi_keys
+    driver.configure("remove_keys key1, key2\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    driver.run
+    assert(!index_cmds[1].has_key?('key1'))
+    assert(!index_cmds[1].has_key?('key2'))
+  end
+
   def test_request_error
     stub_elastic_ping
     stub_elastic_unavailable

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2016-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -124,6 +124,7 @@ files:
 - Gemfile
 - History.md
 - LICENSE.txt
+- PULL_REQUEST_TEMPLATE.md
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec