fluent-plugin-elasticsearch 1.10.1 → 1.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6924c1c7c90f7eb629011bbae3db822799584200
-  data.tar.gz: 52ab17e9b443275e215de285327be011881681b9
+  metadata.gz: 7756405c1e59422b26624fbd6e3a0fded30c6d46
+  data.tar.gz: c12bd57fb39982e51e9a5a4dfaa73f3ef5da3e2c
 SHA512:
-  metadata.gz: 85120102f59c4713273af0706b90e36a696e27329c63a3f748762c626633f02fe22ae60ebd6819c9f9819dcc4fa5d05d5df7ed58f2d4e43a3e2625be7599559d
-  data.tar.gz: 4f0a559e177c0fccaf0e1bb916decf0226508b49af463448556b76622c5c9ba064246052da1b92fb876d36ce77f0e3644f18144f3cd8b7e72fc7fae6ed70737d
+  metadata.gz: 7fb789e36c7a2ad8ae823fa4ed0f7578293bd70426629fccdb09681cc82bbc4e7183439dee046a117bb08ea7de4bcc69eed8052822e5c454e1934218db8d23f2
+  data.tar.gz: ea0983374a67d7f23ad2e5c2a043de495131784b6b583abbd12d483946dc13aa9fc01e9601fabded84ef11404efd23643f280ff9623aafdb4ce5d1a02f8db43a

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.10.2
+- backport adding `include_timestamp` option (#311)
+
 ### 1.10.1
 - backport escaping basic authentication user information placeholders (#309)
 - backport handling dynamic config misconfiguration (#308)

data/README.md

@@ -26,6 +26,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [time_precision](#time_precision)
   + [time_key](#time_key)
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
+  + [include_timestamp](#time_key_exclude_timestamp)
   + [utc_index](#utc_index)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
@@ -130,6 +131,14 @@ logstash_format true # defaults to false
 
 This is meant to make writing data into ElasticSearch indices compatible to what [Logstash](https://www.elastic.co/products/logstash) calls them. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana). See logstash_prefix and logstash_dateformat to customize this index name pattern. The index name will be `#{logstash_prefix}-#{formated_date}`
 
+### include_timestamp
+
+```
+include_timestamp true # defaults to false
+```
+
+Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
+
 ### logstash_prefix
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.10.1'
+  s.version       = '1.10.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -28,6 +28,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :target_type_key, :string, :default => nil
   config_param :time_key_format, :string, :default => nil
   config_param :time_precision, :integer, :default => 0
+  config_param :include_timestamp, :bool, :default => false
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
   config_param :logstash_prefix_separator, :string, :default => '-'
@@ -310,10 +311,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
         record = flatten_record(record)
       end
 
-      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
-      if target_index_parent && target_index_parent[target_index_child_key]
-        target_index = target_index_parent.delete(target_index_child_key)
-      elsif @logstash_format
+      dt = nil
+      if @logstash_format || @include_timestamp
         if record.has_key?(TIMESTAMP_FIELD)
           rts = record[TIMESTAMP_FIELD]
           dt = parse_time(rts, time, tag)
@@ -325,6 +324,12 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
           dt = Time.at(time).to_datetime
           record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
         end
+      end
+
+      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+      if target_index_parent && target_index_parent[target_index_child_key]
+        target_index = target_index_parent.delete(target_index_child_key)
+      elsif @logstash_format
         dt = dt.new_offset(0) if @utc_index
         target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
       else

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -7,7 +7,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
 
   config_param :delimiter, :string, :default => "."
 
-  DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
+  DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
   DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
 
   attr_reader :dynamic_config
@@ -139,7 +139,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         next
       end
 
-      if eval_or_val(dynamic_conf['logstash_format'])
+      if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
         if record.has_key?("@timestamp")
           time = Time.parse record["@timestamp"]
         elsif record.has_key?(dynamic_conf['time_key'])
@@ -148,7 +148,9 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         else
           record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
         end
+      end
 
+      if eval_or_val(dynamic_conf['logstash_format'])
         if eval_or_val(dynamic_conf['utc_index'])
           target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
         else

data/test/plugin/test_out_elasticsearch.rb

@@ -755,7 +755,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil(index_cmds[1]['@timestamp'])
   end
 
-  def test_adds_logstash_timestamp_when_configured
+  def test_adds_timestamp_when_logstash
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
@@ -766,6 +766,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_adds_timestamp_when_include_timestamp
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.now
+    time = ts.to_time
+    driver.emit(sample_record, time)
+    driver.run
+    tf = "%Y-%m-%dT%H:%M:%S%:z"
+    timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
+  end
+
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
     stub_elastic_ping
@@ -777,6 +791,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_timestamp_when_included_in_record_without_logstash
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   def test_uses_custom_time_key
     driver.configure("logstash_format true
                       time_key vtm\n")
@@ -803,6 +828,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
   end
 
+  def test_uses_custom_time_key_with_format_without_logstash
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key_format %Y-%m-%d %H:%M:%S.%N%z
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "2001-02-03 13:14:01.673+02:00"
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal("test", index_cmds[0]['index']['_index'])
+  end
+
   def test_uses_custom_time_key_exclude_timekey
     driver.configure("logstash_format true
                       time_key vtm
@@ -828,6 +868,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_format_without_logstash
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "2001-02-03T13:14:01.673+02:00"
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert_equal("test", index_cmds[0]['index']['_index'])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   data(:default => nil,
        :custom_tag => 'es_plugin.output.time.error')
   def test_uses_custom_time_key_format_logs_an_error(tag_for_error)

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -232,7 +232,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('fluentd', index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_speficied_index
+  def test_writes_to_specified_index
     driver.configure("index_name myindex\n")
     stub_elastic_ping
     stub_elastic
@@ -241,7 +241,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_speficied_index_uppercase
+  def test_writes_to_specified_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
     stub_elastic
@@ -250,7 +250,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_speficied_type
+  def test_writes_to_specified_type
     driver.configure("type_name mytype\n")
     stub_elastic_ping
     stub_elastic
@@ -259,7 +259,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_speficied_host
+  def test_writes_to_specified_host
     driver.configure("host 192.168.33.50\n")
     stub_elastic_ping("http://192.168.33.50:9200")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
@@ -268,7 +268,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
-  def test_writes_to_speficied_port
+  def test_writes_to_specified_port
     driver.configure("port 9201\n")
     stub_elastic_ping("http://localhost:9201")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
@@ -419,6 +419,17 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_uses_custom_timestamp_when_included_in_record
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_timestamp_when_included_in_record_logstash
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
@@ -429,7 +440,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
-  def test_uses_custom_time_key
+  def test_uses_custom_time_key_logstash
     driver.configure("logstash_format true
                       time_key vtm\n")
     stub_elastic_ping
@@ -441,7 +452,45 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_timestamp
+    driver.configure("include_timestamp true
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_time_key_timestamp_custom_index
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal('test', index_cmds.first['index']['_index'])
+  end
+
   def test_uses_custom_time_key_exclude_timestamp
+    driver.configure("include_timestamp true
+                      time_key vtm
+                      time_key_exclude_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
+  end
+
+  def test_uses_custom_time_key_exclude_timestamp_logstash
     driver.configure("logstash_format true
                       time_key vtm
                       time_key_exclude_timestamp true\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.10.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-16 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd