fluent-plugin-elasticsearch 2.0.0.rc.6 → 2.0.0.rc.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac60b728ec23b3ba9a24a87f1b58fcf3dbf2376f
-  data.tar.gz: fe9d1c7d1f5c07d84a69e27bc534c6c6db5512b6
+  metadata.gz: f225504787161485e7c1623d16d248cfddbccb0e
+  data.tar.gz: 208b3f560dfb6f76c8154abd5aa935cd488330ef
 SHA512:
-  metadata.gz: 51b2bf67047821e1d028acdc507d9f1f235444cd6ffa2d0f6dded6284231ef86473aed7e2e448daa89af31ff3a6cbb2a05aa17a36be5a8717a9a1473ad05941d
-  data.tar.gz: b5a8317b90ddd6b52a8c6cdd2fddf7f609682422bcd24308332d1a760acf990851fb3bfde16d7c6ccd7c430aeb0f1b03186e163c21b35d4a00f4b1b87242ad57
+  metadata.gz: 643760687fe81204c27d67b31a30538ec8d49baca9bc96797c836c8029fa06ca2d4fb0610f326ccf82a1ae047ad92429b5ef3a8319ce05f93a8dd2105136993b
+  data.tar.gz: 0bfce17925d11c35c658c3439cf3888358b2e3699e91351c149871357916b3165c45a0750203585fa66a4175b288d0008b40b4c11530e6b3cdb0509ca23bde0f

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.0.0.rc.7
+- Add `include_timestamp` option (#310)
+
 ### 2.0.0.rc.6
 - Improve documentation (#304)
 - Handle dynamic_config misconfigurations (#305)

data/README.md

@@ -28,6 +28,7 @@ Current maintainers: @cosmo0920
   + [time_precision](#time_precision)
   + [time_key](#time_key)
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
+  + [include_timestamp](#time_key_exclude_timestamp)
   + [utc_index](#utc_index)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
@@ -147,6 +148,14 @@ This is meant to make writing data into ElasticSearch indices compatible to what
 
 :warning: Setting this option to `true` will ignore the `index_name` setting. The default index name prefix is `logstash-`.
 
+### include_timestamp
+
+```
+include_timestamp true # defaults to false
+```
+
+Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
+
 ### logstash_prefix
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.0.0.rc.6'
+  s.version       = '2.0.0.rc.7'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -33,6 +33,7 @@ module Fluent::Plugin
     config_param :target_type_key, :string, :default => nil
     config_param :time_key_format, :string, :default => nil
     config_param :time_precision, :integer, :default => 9
+    config_param :include_timestamp, :bool, :default => false
     config_param :logstash_format, :bool, :default => false
     config_param :logstash_prefix, :string, :default => "logstash"
     config_param :logstash_prefix_separator, :string, :default => '-'
@@ -339,10 +340,8 @@ module Fluent::Plugin
           record = flatten_record(record)
         end
 
-        target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
-        if target_index_parent && target_index_parent[target_index_child_key]
-          target_index = target_index_parent.delete(target_index_child_key)
-        elsif @logstash_format
+        dt = nil
+        if @logstash_format || @include_timestamp
           if record.has_key?(TIMESTAMP_FIELD)
             rts = record[TIMESTAMP_FIELD]
             dt = parse_time(rts, time, tag)
@@ -354,6 +353,12 @@ module Fluent::Plugin
             dt = Time.at(time).to_datetime
             record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
           end
+        end
+
+        target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+        if target_index_parent && target_index_parent[target_index_child_key]
+          target_index = target_index_parent.delete(target_index_child_key)
+        elsif @logstash_format
           dt = dt.new_offset(0) if @utc_index
           target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
         else

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -10,7 +10,7 @@ module Fluent::Plugin
 
     config_param :delimiter, :string, :default => "."
 
-    DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
+    DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
     DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
 
     attr_reader :dynamic_config
@@ -148,7 +148,7 @@ module Fluent::Plugin
           next
         end
 
-        if eval_or_val(dynamic_conf['logstash_format'])
+        if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
           if record.has_key?("@timestamp")
             time = Time.parse record["@timestamp"]
           elsif record.has_key?(dynamic_conf['time_key'])
@@ -157,7 +157,9 @@ module Fluent::Plugin
           else
             record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
           end
+        end
 
+        if eval_or_val(dynamic_conf['logstash_format'])
           if eval_or_val(dynamic_conf['utc_index'])
             target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
           else

data/test/plugin/test_out_elasticsearch.rb

@@ -940,7 +940,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil(index_cmds[1]['@timestamp'])
   end
 
-  def test_adds_logstash_timestamp_when_configured
+  def test_adds_timestamp_when_logstash
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
@@ -955,6 +955,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
   end
 
+  def test_adds_timestamp_when_include_timestamp
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.now
+    time = Fluent::EventTime.from_time(ts.to_time)
+    driver.run(default_tag: 'test') do
+      driver.feed(time, sample_record)
+    end
+    tf = "%Y-%m-%dT%H:%M:%S%:z"
+    timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
+  end
+
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
     stub_elastic_ping
@@ -967,6 +982,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_timestamp_when_included_in_record_without_logstash
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   def test_uses_custom_time_key
     driver.configure("logstash_format true
                       time_key vtm\n")
@@ -995,6 +1022,22 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
   end
 
+  def test_uses_custom_time_key_with_format_without_logstash
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key_format %Y-%m-%d %H:%M:%S.%N%z
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "2001-02-03 13:14:01.673+02:00"
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal("test", index_cmds[0]['index']['_index'])
+  end
+
   def test_uses_custom_time_key_exclude_timekey
     driver.configure("logstash_format true
                       time_key vtm
@@ -1022,6 +1065,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_format_without_logstash
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "2001-02-03T13:14:01.673+02:00"
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
+    assert_equal("test", index_cmds[0]['index']['_index'])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   data(:default => nil,
        :custom_tag => 'es_plugin.output.time.error')
   def test_uses_custom_time_key_format_logs_an_error(tag_for_error)

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -236,7 +236,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('fluentd', index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_speficied_index
+  def test_writes_to_specified_index
     driver.configure("index_name myindex\n")
     stub_elastic_ping
     stub_elastic
@@ -246,7 +246,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_speficied_index_uppercase
+  def test_writes_to_specified_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
     stub_elastic
@@ -256,7 +256,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_speficied_type
+  def test_writes_to_specified_type
     driver.configure("type_name mytype\n")
     stub_elastic_ping
     stub_elastic
@@ -266,7 +266,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_speficied_host
+  def test_writes_to_specified_host
     driver.configure("host 192.168.33.50\n")
     stub_elastic_ping("http://192.168.33.50:9200")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
@@ -276,7 +276,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
-  def test_writes_to_speficied_port
+  def test_writes_to_specified_port
     driver.configure("port 9201\n")
     stub_elastic_ping("http://localhost:9201")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
@@ -438,6 +438,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_uses_custom_timestamp_when_included_in_record
+    driver.configure("include_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_timestamp_when_included_in_record_logstash
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
@@ -449,7 +461,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
-  def test_uses_custom_time_key
+  def test_uses_custom_time_key_logstash
     driver.configure("logstash_format true
                       time_key vtm\n")
     stub_elastic_ping
@@ -462,7 +474,48 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_timestamp
+    driver.configure("include_timestamp true
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_time_key_timestamp_custom_index
+    driver.configure("include_timestamp true
+                      index_name test
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal('test', index_cmds.first['index']['_index'])
+  end
+
   def test_uses_custom_time_key_exclude_timestamp
+    driver.configure("include_timestamp true
+                      time_key vtm
+                      time_key_exclude_timestamp true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
+    assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
+  end
+
+  def test_uses_custom_time_key_exclude_timestamp_logstash
     driver.configure("logstash_format true
                       time_key vtm
                       time_key_exclude_timestamp true\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc.6
+  version: 2.0.0.rc.7
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-16 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd