fluent-plugin-elasticsearch 2.0.0.rc.4 → 2.0.0.rc.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4288ea76d01f51dfa10c773ff4316d4e3b0743b7
-  data.tar.gz: 2e0a2f59fe7edb11b78a22f8d3a8010a0138a787
+  metadata.gz: 9cc6bc3f6b89a0735bf96c5c0575d39e58458746
+  data.tar.gz: ee98c63584d8249795b5e4c1e1a75fcc2059b2ab
 SHA512:
-  metadata.gz: 0cf2267e5ea28e049cc12b8132ec46535e2216f0954d26f1afa30c70e7933bae4dac520dcd00342c467617e8b9e3b3ad32ddf8d0c65dde8090b35b9ff782b40e
-  data.tar.gz: b8cf803a896d5cbcea20cae5c399e2cb91943c88e4e126cfc427007d3654906608e7ae1ba64b490c59a548cc2a19e484e80ac9584e5915647489c8431eb7e7c8
+  metadata.gz: eb6c6cb81709dc0b7aa2845e64346621db2cfed45ff16d75cb2384e669cfe8e0497efb8b1d36ea94dbc82da40397347662c118afeaef91347f30b73d93ec940f
+  data.tar.gz: e7eef17ef1e8cab63f79baebba312410455c997c1a5a2128416b48b5bf26c47320c93835e70167b8e218dc022b9171b9d7e802653e205c76dfc81018cfbf3689

data/History.md

@@ -4,8 +4,12 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.0.0.rc.5
+- make configurable with `ssl_version` parameter (#299)
+- add `logstash_prefix_separator` config parameter (#297)
+
 ### 2.0.0.rc.4
-- fix license identifier in gemspec
+- fix license identifier in gemspec (#294)
 
 ### 2.0.0.rc.3
 - add built-in placeholders support (#288, #293)

data/ISSUE_TEMPLATE.md

@@ -9,3 +9,8 @@ Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
 **OR**
 
 Provide example config and message
+
+#### Using Fluentd and ES plugin versions
+
+* Fluentd v0.12 or v0.14
+* ES plugin 2.x.y or 1.x.y

data/README.md

@@ -21,6 +21,7 @@ Current maintainers: @cosmo0920
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
   + [logstash_format](#logstash_format)
   + [logstash_prefix](#logstash_prefix)
+  + [logstash_prefix_separator](#logstash_prefix_separator)
   + [logstash_dateformat](#logstash_dateformat)
   + [pipeline](#pipeline)
   + [time_key_format](#time_key_format)
@@ -49,7 +50,7 @@ Current maintainers: @cosmo0920
   + [reconnect_on_error](#reconnect_on_error)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
-  + [Buffered output options](#buffered-output-options)
+  + [Buffer options](#buffer-options)
   + [Hash flattening](#hash-flattening)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
@@ -67,6 +68,7 @@ Current maintainers: @cosmo0920
 |  < 2.0.0 | >= v0.12.0 | >= 1.9 |
 
 NOTE: fluent-plugin-elasticsearch v2.0.0 is now RC. We will release stable v2.0.0 soon.
+
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
 ## Installation
@@ -136,6 +138,12 @@ This is meant to make writing data into ElasticSearch indices compatible to what
 logstash_prefix mylogs # defaults to "logstash"
 ```
 
+### logstash_prefix_separator
+
+```
+logstash_prefix_separator _ # defaults to "-"
+```
+
 ### logstash_dateformat
 
 The strftime format to generate index target index name when `logstash_format` is set to true. By default, the records are inserted into index `logstash-YYYY.MM.DD`. This option, alongwith `logstash_prefix` lets us insert into specified index like `mylogs-YYYYMM` for a monthly index.
@@ -455,13 +463,18 @@ client_key /path/to/your/private/key
 client_key_pass password
 ```
 
+If you want to configure SSL/TLS version, you can specify ssl\_version parameter.
+```
+ssl_version TLSv1_2 # or [SSLv23, TLSv1, TLSv1_1]
+```
+
 ### Proxy Support
 
 Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
 
-### Buffered output options
+### Buffer options
 
-`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Buffered Output plugin](http://docs.fluentd.org/articles/buffer-plugin-overview). It adds the following options:
+`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v0.14/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
 
 ```
 buffer_type memory
@@ -473,6 +486,8 @@ num_threads 1
 
 The value for option `buffer_chunk_limit` should not exceed value `http.max_content_length` in your Elasticsearch setup (by default it is 100mb).
 
+**Note**: If you use or evaluate Fluentd v0.14, you can use `<buffer>` directive to specify buffer configuration, too. In more detail, please refer to the [buffer configuration options for v0.14](https://docs.fluentd.org/v0.14/articles/buffer-plugin-overview#configuration-parameters)
+
 ### Hash flattening
 
 Elasticsearch will complain if you send object and concrete values to the same field. For example, you might have logs that look this, from different places:
@@ -512,6 +527,8 @@ Alternatively, consider using [fluent-plugin-forest](https://github.com/tagomori
 
 And yet another option is described in Dynamic Configuration section.
 
+**Note**: If you use or evaluate Fluentd v0.14, you can use builtin placeholders. In more detail, please refer to [Placeholders](#placeholders) section.
+
 ### Dynamic configuration
 
 If you want configurations to depend on information in messages, you can use `elasticsearch_dynamic`. This is an experimental variation of the ElasticSearch plugin allows configuration values to be specified in ways such as the below:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.0.0.rc.4'
+  s.version       = '2.0.0.rc.5'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -35,6 +35,7 @@ module Fluent::Plugin
     config_param :time_precision, :integer, :default => 9
     config_param :logstash_format, :bool, :default => false
     config_param :logstash_prefix, :string, :default => "logstash"
+    config_param :logstash_prefix_separator, :string, :default => '-'
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
     config_param :type_name, :string, :default => "fluentd"
@@ -54,6 +55,7 @@ module Fluent::Plugin
     config_param :client_cert, :string, :default => nil
     config_param :client_key_pass, :string, :default => nil
     config_param :ca_file, :string, :default => nil
+    config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1
     config_param :remove_keys, :string, :default => nil
     config_param :remove_keys_on_update, :string, :default => ""
     config_param :remove_keys_on_update_key, :string, :default => nil
@@ -170,7 +172,7 @@ module Fluent::Plugin
                                                                               transport_options: {
                                                                                 headers: { 'Content-Type' => 'application/json' },
                                                                                 request: { timeout: @request_timeout },
-                                                                                ssl: { verify: @ssl_verify, ca_file: @ca_file }
+                                                                                ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
                                                                               }
                                                                             }), &adapter_conf)
         es = Elasticsearch::Client.new transport: transport
@@ -334,7 +336,7 @@ module Fluent::Plugin
             record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
           end
           dt = dt.new_offset(0) if @utc_index
-          target_index = "#{logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
+          target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
         else
           target_index = index_name
         end

data/test/plugin/test_out_elasticsearch.rb

@@ -74,6 +74,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '/es/', instance.path
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
+    assert_equal :TLSv1, instance.ssl_version
+    assert_nil instance.client_key
+    assert_nil instance.client_cert
+    assert_nil instance.client_key_pass
   end
 
   test 'lack of tag in chunk_keys' do
@@ -748,6 +752,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
+  def test_writes_to_logstash_index_with_specified_prefix_and_separator
+    separator = '_'
+    driver.configure("logstash_format true
+                      logstash_prefix_separator #{separator}
+                      logstash_prefix myprefix")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
   class LogStashPrefixPlaceholdersTest < self
     def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
       driver.configure("logstash_format true

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -69,6 +69,10 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
     assert_equal '/es/', instance.path
+    assert_equal :TLSv1, instance.ssl_version
+    assert_nil instance.client_key
+    assert_nil instance.client_cert
+    assert_nil instance.client_key_pass
   end
 
   def test_defaults

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc.4
+  version: 2.0.0.rc.5
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-13 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd