fluent-plugin-elasticsearch 1.9.4 → 1.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec35bcfd663fc3e9c9701ce0cb416916294d5688
-  data.tar.gz: 45e2ff5e1d5414fa57463dbaa524e9e7c25bcfd3
+  metadata.gz: 8af381642b4168628152b2baf959f30d85ecb3aa
+  data.tar.gz: 64f4f74cf949f999db949489a59ea264d524b41e
 SHA512:
-  metadata.gz: 621d5699f57f1adb03f347d11598d61b3c7ea2a8a1791865b8dfe242d8abc8394b27df4756eb604f883fdd743d35c3443a847938575f25152b55529fd101f501
-  data.tar.gz: 97e7287bb4f8c553206521d0eb88dac9bb9e89d4c118c8bc19a26e7756534c4be6d289b5847274ac0c829ddca69927adf46a1fea1f3c477d364fa0fae4e93e65
+  metadata.gz: 3b1da6085fb920f79bf1fd1951da9aa00fea43ad78cf1dc48febb51973cc075e6d6339b050254cd5f8eb5e08c5ec77db2ced1140abd0c3e918aeced9f560a7fb
+  data.tar.gz: e8701fbad7986ddb9be19659a6d27d79b180b5ec25ca95384bd888af0edef5e3b9ea112ad82fd905f7dab6da2992006dda3234caf3875bacea6f5eab3ef81bee

data/History.md

@@ -4,6 +4,12 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.9.5
+- sub-second time precision [(#249)](https://github.com/uken/fluent-plugin-elasticsearch/pull/249)
+
+### 1.9.4
+- Include 'Content-Type' header in `transport_options`
+
 ### 1.9.2
 - Fix elasticsearch_dynamic for v0.14 (#224)
 

data/README.md

@@ -21,6 +21,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [logstash_prefix](#logstash_prefix)
   + [logstash_dateformat](#logstash_dateformat)
   + [time_key_format](#time_key_format)
+  + [time_precision](#time_precision)
   + [time_key](#time_key)
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
   + [utc_index](#utc_index)
@@ -140,6 +141,12 @@ For example to parse ISO8601 times with sub-second precision:
 time_key_format %Y-%m-%dT%H:%M:%S.%N%z
 ```
 
+### time_precision
+
+Should the record not include a `time_key`, define the degree of sub-second time precision to preserve from the `time` portion of the routed event.
+
+For example, should your input plugin not include a `time_key` in the record but it able to pass a `time` to the router when emitting the event (AWS CloudWatch events are an example of this), then this setting will allow you to preserve the sub-second time resolution of those events. This is the case for: [fluent-plugin-cloudwatch-ingest](https://github.com/sampointer/fluent-plugin-cloudwatch-ingest).
+
 ### time_key
 
 By default, when inserting records in [Logstash](https://www.elastic.co/products/logstash) format, `@timestamp` is dynamically created with the time at log ingestion. If you'd like to use a custom time, include an `@timestamp` with your record.
@@ -360,7 +367,7 @@ remove_keys a_parent, a_routing # a_parent and a_routing fields won't be sent to
 
 ### remove_keys_on_update
 
-Remove keys on update will not update the configured keys in elasticsearch when a record is being updated.  
+Remove keys on update will not update the configured keys in elasticsearch when a record is being updated.
 This setting only has any effect if the write operation is update or upsert.
 
 If the write setting is upsert then these keys are only removed if the record is being

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.9.4'
+  s.version       = '1.9.5'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -27,6 +27,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :target_index_key, :string, :default => nil
   config_param :target_type_key, :string, :default => nil
   config_param :time_key_format, :string, :default => nil
+  config_param :time_precision, :integer, :default => 0
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
   config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
@@ -300,7 +301,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
           record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
         else
           dt = Time.at(time).to_datetime
-          record[TIMESTAMP_FIELD] = dt.to_s
+          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
         end
         dt = dt.new_offset(0) if @utc_index
         target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"

data/test/plugin/test_out_elasticsearch.rb

@@ -120,8 +120,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver('test', config)
-  end 
-  
+  end
+
 
   def test_template_create_invalid_filename
     config = %{
@@ -177,14 +177,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
     stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
       to_return(:status => 200, :body => "", :headers => {})
-    
+
     driver('test', config)
-    
+
     assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
     assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
     assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
   end
-  
+
   def test_templates_not_used
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
@@ -199,7 +199,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       template_name   logstash
       template_file   #{template_file}
       templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
-    }    
+    }
     # connection start
     stub_request(:head, "https://john:doe@logs.google.com:777/es//").
       to_return(:status => 200, :body => "", :headers => {})
@@ -254,7 +254,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_raise(RuntimeError) {
       driver('test', config)
     }
-    
+
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
     assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
   end
@@ -780,6 +780,37 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_no_subsecond_precision_by_default
+    driver.configure("logstash_format true\n")
+    stub_elastic_ping
+    stub_elastic
+    begin
+      time = Fluent::EventTime.new(Time.now.to_i, 000000000)
+    rescue
+      time = Fluent::Engine.now
+    end
+    driver.emit(sample_record, time)
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601)
+  end
+
+  def test_uses_subsecond_precision_when_configured
+    driver.configure("logstash_format true
+                      time_precision 3\n")
+    stub_elastic_ping
+    stub_elastic
+    begin
+    time = Fluent::EventTime.new(Time.now.to_i, 000000000)
+    rescue
+      time = Fluent::Engine.now
+    end
+    driver.emit(sample_record, time)
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
+  end
+
   def test_doesnt_add_tag_key_by_default
     stub_elastic_ping
     stub_elastic
@@ -941,7 +972,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
-    
+
     driver.configure("reconnect_on_error true\n")
     driver.emit(sample_record)
 
@@ -965,7 +996,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
-    
+
     driver.configure("reconnect_on_error false\n")
     driver.emit(sample_record)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.9.4
+  version: 1.9.5
 platform: ruby
 authors:
 - diogo