fluent-plugin-elasticsearch 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e5c36c889794052ff9bab9d9b6e69aa8d937eed
-  data.tar.gz: a719c42134c8d53969abe517a47f47a6936d092d
+  metadata.gz: 89a28c84005c362158da08a71ea7c10036df254d
+  data.tar.gz: 87a5e6283b6e3bd057ec949ee95316460bc8f6da
 SHA512:
-  metadata.gz: e75bdccd2859b063eb1fc48fdce8394e43648ff05b4d25214b66b85111482c1d8d671e24d3a5376d37f0f1093fbefb2866afc5e9882728df2b09c2147cf16a90
-  data.tar.gz: 3393dce7f111c75ce16b8b4af7fff51064870cb1033bf446701180cf0c84458ffc3a8ea3f6a58503eff46d1d7e414e588073497fd5e6a20becc036971ddb40d8
+  metadata.gz: fde384e1b97967886d1446785e1cdb5eedb1bf7f98ab7841d321d34886dc84ee95f255d8d86b289c03b40e469528efce6a288b63072a6fd9f40f5451b82bd043
+  data.tar.gz: 35d51c4cc3508cf12e6c4971db4ab4c947c4688cc4f021a33c4d0bf1f325a03a013b1b1b30195be8a64296636997e757a518fdd5516234741824cec20151197b

data/.travis.yml

@@ -5,6 +5,7 @@ rvm:
   - 2.0.0
   - 2.1
   - 2.2
+  - 2.3.0
 
 script: bundle exec rake test
 sudo: false

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem 'simplecov', require: false
 gem 'coveralls', require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/

data/History.md

@@ -1,5 +1,11 @@
 ## Changelog
 
+### Future
+
+### 1.4.0
+- add `target_index_key` to specify target index (#153)
+- add `time_key_format` for faster time format parsing (#154)
+
 ### 1.3.0
 - add `write_operation`
 

data/README.md

@@ -20,8 +20,10 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [logstash_format](#logstash_format)
   + [logstash_prefix](#logstash_prefix)
   + [logstash_dateformat](#logstash_dateformat)
+  + [time_key_format](#time_key_format)
   + [time_key](#time_key)
   + [utc_index](#utc_index)
+  + [target_index_key](#target_index_key)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
@@ -30,6 +32,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [id_key](#id_key)
   + [write_operation](#write_operation)
   + [Client/host certificate options](#clienthost-certificate-options)
+  + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
@@ -45,11 +48,11 @@ $ gem install fluent-plugin-elasticsearch
 
 ## Usage
 
-In your Fluentd configuration, use `type elasticsearch`. Additional configuration is optional, default values would look like this:
+In your Fluentd configuration, use `@type elasticsearch`. Additional configuration is optional, default values would look like this:
 
 ```
 <match my.logs>
-  type elasticsearch
+  @type elasticsearch
   host localhost
   port 9200
   index_name fluentd
@@ -96,7 +99,7 @@ Specify `ssl_verify false` to skip ssl verification (defaults to true)
 logstash_format true # defaults to false
 ```
 
-This is meant to make writing data into ElasticSearch compatible to what [Logstash](https://www.elastic.co/products/logstash) writes. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana).
+This is meant to make writing data into ElasticSearch indices compatible to what [Logstash](https://www.elastic.co/products/logstash) calls them. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana). See logstash_prefix and logstash_dateformat to customize this index name pattern. The index name will be `#{logstash_prefix}-#{formated_date}`
 
 ### logstash_prefix
 
@@ -106,12 +109,24 @@ logstash_prefix mylogs # defaults to "logstash"
 
 ### logstash_dateformat
 
-By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `mylogs-YYYYMM` for a monthly index.
+The strftime format to generate index target index name when `logstash_format` is set to true. By default, the records are inserted into index `logstash-YYYY.MM.DD`. This option, alongwith `logstash_prefix` lets us insert into specified index like `mylogs-YYYYMM` for a monthly index.
 
 ```
 logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
 ```
 
+### time_key_format
+
+The format of the time stamp field (`@timestamp` or what you specify with [time_key][#time_key]). This parameter only has an effect when [logstash_format][#logstash_format] is true as it only affects the name of the index we write to. Please see [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime) for information about the value of this format.
+
+Setting this to a known format can vastly improve your log ingestion speed if all most of your logs are in the same format. If there is an error parsing this format the timestamp will default to the ingestion time. If you are on Ruby 2.0 or later you can get a further performance improvment by installing the "strptime" gem: `fluent-gem install strptime`.
+
+For example to parse ISO8601 times with sub-second precision:
+
+```
+time_key_format %Y-%m-%dT%H:%M:%S.%N%z
+```
+
 ### time_key
 
 By default, when inserting records in [Logstash](https://www.elastic.co/products/logstash) format, `@timestamp` is dynamically created with the time at log ingestion. If you'd like to use a custom time, include an `@timestamp` with your record.
@@ -141,7 +156,7 @@ The output will be
 ```
 {
   "title": "developer",
-  "@timstamp": "2014-12-19T08:01:03Z",
+  "@timestamp": "2014-12-19T08:01:03Z",
   "vtm": "2014-12-19T08:01:03Z"
 }
 ```
@@ -154,6 +169,39 @@ utc_index true
 
 By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
 
+### target_index_key
+
+Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms.
+
+If it is present in the record (and the value is non falsey) the value will be used as the index name to write to and then removed from the record before output; if it is not found then it will use logstash_format or index_name settings as configured.
+
+Suppose you have the following settings
+
+```
+target_index_key @target_index
+index_name fallback
+```
+
+If your input is:
+```
+{
+  "title": "developer",
+  "@timestamp": "2014-12-19T08:01:03Z",
+  "@target_index": "logstash-2014.12.19"
+}
+```
+
+The output would be
+
+```
+{
+  "title": "developer",
+  "@timestamp": "2014-12-19T08:01:03Z",
+}
+```
+
+and this record will be written to the specified index (`logstash-2014.12.19`) rather than `fallback`.
+
 ### request_timeout
 
 You can specify HTTP request timeout.
@@ -200,7 +248,7 @@ This will add the Fluentd tag in the JSON record. For instance, if you have a co
 
 ```
 <match my.logs>
-  type elasticsearch
+  @type elasticsearch
   include_tag_key true
   tag_key _key
 </match>
@@ -229,7 +277,7 @@ This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will tr
 
 ### write_operation
 
-The write_operation can be any of: 
+The write_operation can be any of:
 
 | Operation | Description          |
 | ------------- | ----------- |
@@ -254,6 +302,10 @@ client_key /path/to/your/private/key
 client_key_pass password
 ```
 
+### Proxy Support
+
+Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
+
 ### Buffered output options
 
 `fluentd-plugin-elasticsearch` extends [Fluentd's builtin Buffered Output plugin](http://docs.fluentd.org/articles/buffer-plugin-overview). It adds the following options:
@@ -276,7 +328,7 @@ Alternatively, consider using [fluent-plugin-forest](https://github.com/tagomori
 
 ```
 <match my.logs.*>
-  type forest
+  @type forest
   subtype elasticsearch
   remove_prefix my.logs
   <template>
@@ -294,7 +346,7 @@ If you want configurations to depend on information in messages, you can use `el
 
 ```
 <match my.logs.*>
-  type elasticsearch_dynamic
+  @type elasticsearch_dynamic
   hosts ${record['host1']}:9200,${record['host2']}:9200
   index_name my_index.${Time.at(time).getutc.strftime(@logstash_dateformat)}
   logstash_prefix ${tag_parts[3]}
@@ -311,8 +363,12 @@ If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-el
 
 ## Contributing
 
+There are usually a few feature requests, tagged [Easy](https://github.com/uken/fluent-plugin-elasticsearch/issues?q=is%3Aissue+is%3Aopen+label%3Alevel%3AEasy), [Normal](https://github.com/uken/fluent-plugin-elasticsearch/issues?q=is%3Aissue+is%3Aopen+label%3Alevel%3ANormal) and [Hard](https://github.com/uken/fluent-plugin-elasticsearch/issues?q=is%3Aissue+is%3Aopen+label%3Alevel%3AHard). Feel free to work on any one of them.
+
 Pull Requests are welcomed.
 
+[![Pull Request Graph](https://graphs.waffle.io/uken/fluent-plugin-elasticsearch/throughput.svg)](https://waffle.io/uken/fluent-plugin-elasticsearch/metrics)
+
 ## Running tests
 
 Install dev dependencies:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.3.0'
+  s.version       = '1.4.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -3,6 +3,10 @@ require 'date'
 require 'excon'
 require 'elasticsearch'
 require 'uri'
+begin
+  require 'strptime'
+rescue LoadError
+end
 
 class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError; end
@@ -16,6 +20,8 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :path, :string, :default => nil
   config_param :scheme, :string, :default => 'http'
   config_param :hosts, :string, :default => nil
+  config_param :target_index_key, :string, :default => nil
+  config_param :time_key_format, :string, :default => nil
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
   config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
@@ -41,16 +47,50 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
 
   def initialize
     super
+    @time_parser = TimeParser.new(@time_key_format, @router)
   end
 
   def configure(conf)
     super
+    @time_parser = TimeParser.new(@time_key_format, @router)
   end
 
   def start
     super
   end
 
+  # once fluent v0.14 is released we might be able to use
+  # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
+  # [sec,nsec] where as we want something we can call `strftime` on...
+  class TimeParser
+    def initialize(time_key_format, router)
+      @time_key_format = time_key_format
+      @router = router
+      @parser = if time_key_format
+        begin
+          # Strptime doesn't support all formats, but for those it does it's
+          # blazingly fast.
+          strptime = Strptime.new(time_key_format)
+          Proc.new { |value| strptime.exec(value).to_datetime }
+        rescue
+          # Can happen if Strptime doesn't recognize the format; or
+          # if strptime couldn't be required (because it's not installed -- it's
+          # ruby 2 only)
+          Proc.new { |value| DateTime.strptime(value, time_key_format) }
+        end
+      else
+        Proc.new { |value| DateTime.parse(value) }
+      end
+    end
+
+    def parse(value, event_time)
+      @parser.call(value)
+    rescue => e
+      @router.emit_error_event("Fluent::ElasticsearchOutput::TimeParser.error", Fluent::Engine.now, {'time' => event_time, 'format' => @time_key_format, 'value' => value }, e)
+      return Time.at(event_time).to_datetime
+    end
+  end
+
   def client
     @_es ||= begin
       excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
@@ -151,20 +191,21 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
 
     chunk.msgpack_each do |tag, time, record|
       next unless record.is_a? Hash
-      if @logstash_format
+      if @target_index_key && record[@target_index_key]
+        target_index = record.delete @target_index_key
+      elsif @logstash_format
         if record.has_key?("@timestamp")
-          time = Time.parse record["@timestamp"]
+          dt = record["@timestamp"]
+          dt = @time_parser.parse(record["@timestamp"], time)
         elsif record.has_key?(@time_key)
-          time = Time.parse record[@time_key]
+          dt = @time_parser.parse(record[@time_key], time)
           record['@timestamp'] = record[@time_key]
         else
-          record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
-        end
-        if @utc_index
-          target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
-        else
-          target_index = "#{@logstash_prefix}-#{Time.at(time).strftime("#{@logstash_dateformat}")}"
+          dt = Time.at(time).to_datetime
+          record.merge!({"@timestamp" => dt.to_s})
         end
+        dt = dt.new_offset(0) if @utc_index
+        target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
       else
         target_index = @index_name
       end

data/test/plugin/test_out_elasticsearch.rb

@@ -8,6 +8,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     Fluent::Test.setup
     require 'fluent/plugin/out_elasticsearch'
     @driver = nil
+    log = Fluent::Engine.log
+    log.out.logs.slice!(0, log.out.logs.length)
   end
 
   def driver(tag='test', conf='')
@@ -155,6 +157,49 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  def test_writes_to_target_index_key
+    driver.configure("target_index_key @target_index\n")
+    stub_elastic_ping
+    stub_elastic
+    record = sample_record.clone
+    driver.emit(sample_record.merge('@target_index' => 'local-override'))
+    driver.run
+    assert_equal('local-override', index_cmds.first['index']['_index'])
+    assert_nil(index_cmds[1]['@target_index'])
+  end
+
+  def test_writes_to_target_index_key_logstash
+    driver.configure("target_index_key @target_index\n")
+    driver.configure("logstash_format true\n")
+    time = Time.parse Date.today.to_s
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record.merge('@target_index' => 'local-override'), time)
+    driver.run
+    assert_equal('local-override', index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_target_index_key_fallack
+    driver.configure("target_index_key @target_index\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('fluentd', index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_target_index_key_fallack_logstash
+    driver.configure("target_index_key @target_index\n")
+    driver.configure("logstash_format true\n")
+    time = Time.parse Date.today.to_s
+    logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
     stub_elastic_ping
@@ -234,25 +279,29 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_logstash_index
     driver.configure("logstash_format true\n")
-    time = Time.parse Date.today.to_s
-    logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
+    #
+    # This is 1 second past midnight in BST, so the UTC index should be the day before
+    dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
+    logstash_index = "logstash-2015.05.31"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time)
+    driver.emit(sample_record, dt.to_time)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_logstash_utc_index
+  def test_writes_to_logstash_non_utc_index
     driver.configure("logstash_format true
                       utc_index false")
-    time = Time.parse Date.today.to_s
-    utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
+    # When using `utc_index false` the index time will be the local day of
+    # ingestion time
+    time = Date.today.to_time
+    index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
     driver.emit(sample_record, time)
     driver.run
-    assert_equal(utc_index, index_cmds.first['index']['_index'])
+    assert_equal(index, index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_logstash_index_with_specified_prefix
@@ -334,6 +383,55 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+
+  def test_uses_custom_time_key_format
+    driver.configure("logstash_format true
+                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "2001-02-03T13:14:01.673+02:00"
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_time_key_format_logs_an_error
+    driver.configure("logstash_format true
+                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
+    stub_elastic_ping
+    stub_elastic
+
+    ts = "2001/02/03 13:14:01,673+02:00"
+    index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
+
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+
+    log = driver.instance.router.emit_error_handler.log
+    errors = log.out.logs.grep /tag="Fluent::ElasticsearchOutput::TimeParser.error"/
+    assert_equal(1, errors.length, "Error was logged for timestamp parse failure")
+
+    assert_equal(index, index_cmds[0]['index']['_index'])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+
+  def test_uses_custom_time_key_format_obscure_format
+    driver.configure("logstash_format true
+                      time_key_format %a %b %d %H:%M:%S %Z %Y\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = "Thu Nov 29 14:33:20 GMT 2001"
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   def test_doesnt_add_tag_key_by_default
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-05 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector