fluent-plugin-elasticsearch 4.2.1 → 4.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eb6b06e90f827162d669e94fb9a622c09f12b35e0a05f82c961ad870e3f9726
-  data.tar.gz: c7964330b33c36176e4b1238e4ba4f2d55e60e6ee6da4fec6799f25d943ab85f
+  metadata.gz: ed7efe8849b8a7ca1d2cc3c81444e604c0bda824209f9297f0114f01c1ba609a
+  data.tar.gz: f74f8726fa993c3e4bc01b742e97ef4579fb7335809aac2dabce03c2b002e9ad
 SHA512:
-  metadata.gz: fd5d961731f5866a7c841f9cbeb11f8434868988862cb9929af4cfbe174a7ce65c38d125b91dc8952120be2da4b5ad08e89c127f9c65b9e38585185acccd1cdc
-  data.tar.gz: 0c15c5670b0a2a9eda90b6962768c8db7e510e2da4b2434e3cb841d2fa5b15cf52a8f65f35b4236ab01cb8721d640c690f3bf753a4f4f269b2c802d9f2d7a9b6
+  metadata.gz: 6adeee82a86f8217f9e6d259990042330d8a96a3a3ddb03a332a90f4ce90996c99aff7385b05c625eee4c6253b9647ba2bfc7856f51ba306ef7df6917d716fb4
+  data.tar.gz: dec1a1a72c88c2fd4df38e1b57ca883d47c5f28582fb5ba9a9210c882986c5e52a44f4eda03bef708961b461555b1538968b905b346e6782ad1090ca4310747d

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name:  Bug Report
+about: Create a report to help us improve. If you have questions about ES plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+- [ ] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.
+
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
+
+**OR**
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and ES plugin versions
+
+* OS version
+* Bare Metal or within Docker or Kubernetes or others?
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* ES plugin 3.x.y/2.x.y or 1.x.y
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
+* ES version (optional)
+* ES template(s) (optional)

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+
+**Is your feature request related to a problem? Please describe.**
+
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+
+<!-- Add any other context or screenshots about the feature request here. -->

data/.github/workflows/issue-auto-closer.yml

@@ -9,4 +9,4 @@ jobs:
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
-        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)|(.*Is your feature request related to a problem? Please describe.*)|(.*Describe the solution you'd like.*)|(.*Describe alternatives you've considered.*)"

data/History.md

@@ -2,6 +2,28 @@
 
 ### [Unreleased]
 
+### 4.3.3
+- Handle invalid Elasticsearch::Client#info response (#855)
+
+### 4.3.2
+- Use log.debug to dump ES 8.x _type information (#852)
+
+### 4.3.1
+- Unsplit huge records by default (#851)
+
+### 4.3.0
+- Add cloud_id & cloud_auth settings (#850)
+- Fix failing tests on es6 client (#848)
+- feat: retry on transport errors (#846)
+- fix typo (#838)
+- Remove unnecessary nil check (#826)
+
+### 4.2.2
+- Remove unnecessary nil check (#826)
+- Fix ILM rollover index template pattern to apply index_separator (#825)
+- Fix ILM rollover alias creation when a placeholder is used in index_name (#823)
+- Add a note about the pitfalls of per-date indexes used with ILM (#822)
+
 ### 4.2.1
 - Update a broken link (#821)
 - Include chunk_id in records (#820)

data/README.ElasticsearchGenID.md

@@ -17,7 +17,7 @@
 In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
 
 ```
-<source>
+<filter>
   @type elasticsearch_genid
   hash_id_key _hash
   include_tag_in_seed false
@@ -27,7 +27,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   record_keys []
   separator _
   hash_type sha1
-</match>
+</filter>
 ```
 
 ## Configuration

data/README.md

@@ -19,6 +19,8 @@ Current maintainers: @cosmo0920
 * [Configuration](#configuration)
   + [host](#host)
   + [port](#port)
+  + [cloud_id](#cloud_id)
+  + [cloud_auth](#cloud_auth)
   + [emit_error_for_missing_id](#emit_error_for_missing_id)
   + [hosts](#hosts)
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
@@ -187,6 +189,26 @@ port 9201 # defaults to 9200
 
 You can specify Elasticsearch port by this parameter.
 
+### cloud_id
+
+```
+cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng== 
+```
+
+You can specify Elasticsearch cloud_id by this parameter.
+
+If you specify `cloud_id` option then `cloud_auth` option is required.
+If you specify `cloud_id` option, `host`, `port`, `user` and `password` options are ignored.
+
+### cloud_auth
+
+```
+cloud_auth 'elastic:slkjdaooewkd87iqQ2O8EQYV'
+```
+
+You can specify Elasticsearch cloud_auth by this parameter.
+
+
 ### emit_error_for_missing_id
 
 ```
@@ -501,7 +523,7 @@ Specify the application name for the rollover index to be created.
 application_name default # defaults to "default"
 ```
 
-If [enable_ilm](#enable_ilm is set, then this parameter will be in effect otherwise ignored.
+If [enable_ilm](#enable_ilm) is set, then this parameter will be in effect otherwise ignored.
 
 ### template_overwrite
 
@@ -1143,6 +1165,8 @@ And yet another option is described in Dynamic Configuration section.
 
 ### Dynamic configuration
 
+**NOTE**: *`out_elasticsearch_dynamic` will be planned to be marked as deprecated.* Please don't use the new Fluentd configuration. This plugin is maintained for backward compatibility.
+
 If you want configurations to depend on information in messages, you can use `elasticsearch_dynamic`. This is an experimental variation of the Elasticsearch plugin allows configuration values to be specified in ways such as the below:
 
 ```
@@ -1255,7 +1279,7 @@ Default value is `true`.
 
 Configure `bulk_message` request splitting threshold size.
 
-Default value is `20MB`. (20 * 1024 * 1024)
+Default value is `-1`(unlimited).
 
 If you specify this size as negative number, `bulk_message` request splitting feature will be disabled.
 
@@ -1881,6 +1905,16 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+Note that if you create a new set of indexes every day, the elasticsearch ILM policy system will treat each day separately and will always
+maintain a separate active write index for each day.
+
+If you have a rollover based on max_age, it will continue to roll the indexes for prior dates even if no new documents are indexed.  If you want
+to delete indexes after a period of time, the ILM policy will never delete the current write index regardless of its age, so you would need a separate
+system, such as curator, to actually delete the old indexes.
+
+For this reason, if you put the date into the index names with ILM you should only rollover based on size or number of documents and may need to use
+curator to actually delete old indexes.
+
 #### Fixed ILM indices
 
 Also, users can use fixed ILM indices configuration.
@@ -1996,4 +2030,7 @@ Install dev dependencies:
 $ gem install bundler
 $ bundle install
 $ bundle exec rake test
+# To just run the test you are working on:
+$ bundle exec rake test TEST=test/plugin/test_out_elasticsearch.rb TESTOPTS='--verbose --name=test_custom_template_with_rollover_index_create_and_custom_ilm'
+
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.2.1'
+  s.version       = '4.3.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -32,12 +32,13 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true)
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
+    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
+    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries
@@ -73,14 +74,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
 
       log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -91,7 +93,8 @@ module Fluent::ElasticsearchIndexTemplate
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
       log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
@@ -99,14 +102,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
                         inject_ilm_settings_to_template(deflector_alias_name,
                                                         target_index,
                                                         ilm_policy_id,
                                                         get_custom_template(template_file,
-                                                                            customize_template))
+                                                                            customize_template),
+                                                        index_separator)
                       else
                         get_custom_template(template_file, customize_template)
                       end
@@ -127,9 +131,9 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template['index_patterns'] = "#{target_index}-*"
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
     if @use_legacy_template
       template.delete('template') if template.include?('template')
       # Prepare settings Hash
@@ -140,7 +144,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
       end
       template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
-      template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
     else
       # Prepare template.settings Hash
       if !template['template'].key?('settings')
@@ -150,7 +154,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
       end
       template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
-      template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
     end
     template
   end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -77,6 +77,8 @@ module Fluent::Plugin
     config_param :port, :integer, :default => 9200
     config_param :user, :string, :default => nil
     config_param :password, :string, :default => nil, :secret => true
+    config_param :cloud_id, :string, :default => nil
+    config_param :cloud_auth, :string, :default => nil
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
     config_param :hosts, :string, :default => nil
@@ -171,6 +173,7 @@ EOC
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
+    config_param :catch_transport_exception_on_retry, :bool, :default => true
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -259,9 +262,11 @@ EOC
             template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
-        end 
+        end
         if @templates
-          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
             templates_hash_install(@templates, @template_overwrite)
           end
         end
@@ -291,8 +296,14 @@ EOC
         @dump_proc = Yajl.method(:dump)
       end
 
+      raise Fluent::ConfigError, "`cloud_auth` must be present if `cloud_id` is present" if @cloud_id && @cloud_auth.nil?
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
 
+      if @cloud_auth
+        @user = @cloud_auth.split(':', -1)[0]
+        @password = @cloud_auth.split(':', -1)[1]
+      end
+
       if @user && m = @user.match(/%{(?<user>.*)}/)
         @user = URI.encode_www_form_component(m["user"])
       end
@@ -338,7 +349,7 @@ EOC
           @type_name = '_doc'.freeze
         end
         if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-          log.info "Detected ES 8.x or above: This parameter has no effect."
+          log.debug "Detected ES 8.x or above: This parameter has no effect."
           @type_name = nil
         end
       end
@@ -470,7 +481,9 @@ EOC
 
     def handle_last_seen_es_major_version
       if @verify_es_version_at_startup && !dry_run?
-        retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
+        retry_operate(@max_retry_get_es_version,
+                      @fail_on_detecting_es_version_retry_exceed,
+                      @catch_transport_exception_on_retry) do
           detect_es_major_version
         end
       else
@@ -480,7 +493,12 @@ EOC
 
     def detect_es_major_version
       @_es_info ||= client.info
-      unless version = @_es_info.dig("version", "number")
+      begin
+        unless version = @_es_info.dig("version", "number")
+          version = @default_elasticsearch_version
+        end
+      rescue NoMethodError => e
+        log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
         version = @default_elasticsearch_version
       end
       version.to_i
@@ -555,7 +573,17 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
+    def cloud_client
+      Elasticsearch::Client.new(
+        cloud_id: @cloud_id,
+        user: @user,
+        password: @password
+      )
+    end
+
     def client(host = nil, compress_connection = false)
+      return cloud_client if @cloud_id
+
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
@@ -907,7 +935,7 @@ EOC
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
         elsif @last_seen_major_version >=8
-          log.warn "Detected ES 8.x or above: document type will not be used."
+          log.debug "Detected ES 8.x or above: document type will not be used."
           target_type = nil
         end
       else
@@ -917,7 +945,7 @@ EOC
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
         elsif @last_seen_major_version >= 8
-          log.warn "Detected ES 8.x or above: document type will not be used."
+          log.debug "Detected ES 8.x or above: document type will not be used."
           target_type = nil
         else
           target_type = type_name
@@ -982,22 +1010,26 @@ EOC
 
     def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
-        if !@logstash_format && @alias_indexes.include?(deflector_alias)
-          log.debug("Index alias #{deflector_alias} already exists (cached)")
-        elsif !@logstash_format && @template_names.include?(template_name)
-          log.debug("Template name #{template_name} already exists (cached)")
+        if !@logstash_format && (deflector_alias.nil? || (@alias_indexes.include? deflector_alias)) && (@template_names.include? template_name)
+          if deflector_alias
+            log.debug("Index alias #{deflector_alias} and template #{template_name} already exist (cached)")
+          else
+            log.debug("Template #{template_name} already exists (cached)")
+          end
         else
-          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             end
             ilm_policy = @ilm_policies[ilm_policy_id] || {}
             create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
-          @template_names << template_name unless template_name.nil?
+          @template_names << template_name
         end
       end
     end