fluent-plugin-elasticsearch 3.5.6 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91343c74b95ee9133fd8323079dbbdd89e539f07942dd936d7107df9dfc5bf8f
-  data.tar.gz: f51a62e66164d542e7aa2d21cfa6565085e3bf6be1db92cc31adb4ad5709588f
+  metadata.gz: 22474d12620f4cc8c2a2224b19e5e98534bd38d86f9489ace710b50139ea1422
+  data.tar.gz: 85c5043ca7ab0905efb9e2093c3b04d83dadae645e5aa2e580dbd601db77fe88
 SHA512:
-  metadata.gz: 283d651e87233d84bd871df3f1d8b472d91f47dc53953ac47b282cfee654762b0199c7779a398485c24eb89885a01aff5204481a660ed6d7e0fb5243ccbfb2ef
-  data.tar.gz: 6ae7ab7dac0721dd82082e01ca9de3bc969949159d9b263cf53e43d62d01a543cb84ba6992d46189f16ab4b9b1c63de2ac8dd49b45e9778e7a9ec01fa66395c8
+  metadata.gz: f22d3fe7adbb44c8b38f9c61757857937c3293c762dfba6842081e08df65ceddd3c75a0bfbf791c325fa1d82380e6d5b09ed7a3a774f03706a095cb8553323ac
+  data.tar.gz: 5f418e39091ac5bdceb2869604480e9b822c7654ba35da44c0ac636b88148168bee352c8bd7c3197fc5642019ab2a145430b4a88a722902c609457f1930eed2d

data/.travis.yml

@@ -1,12 +1,21 @@
 language: ruby
 
-rvm:
-  - 2.1
-  - 2.2
-  - 2.3
-  - 2.4.5
-  - 2.5.3
-  - 2.6.0
+matrix:
+  include:
+    - rvm: 2.1
+      gemfile: Gemfile
+    - rvm: 2.2
+      gemfile: Gemfile
+    - rvm: 2.3
+      gemfile: Gemfile
+    - rvm: 2.4.6
+      gemfile: Gemfile
+    - rvm: 2.5.5
+      gemfile: Gemfile
+    - rvm: 2.6.3
+      gemfile: gemfiles/Gemfile.ilm
+    - rvm: 2.6.3
+      gemfile: Gemfile
 
 gemfile:
  - Gemfile

data/History.md

@@ -1,6 +1,12 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 3.6.0
+- Set order in newly created templates (#660)
+- Merge Support index lifecycle management into master (#659)
+- Support template installation with host placeholder (#654)
+- Support index lifecycle management (#651)
+
 ### 3.5.6
 - Support elasticsearch8 removal of mapping types (#656)
 - Upgrade webmock to 3 (#652)

data/README.md

@@ -87,6 +87,9 @@ Current maintainers: @cosmo0920
   + [ignore_exceptions](#ignore_exceptions)
   + [exception_backup](#exception_backup)
   + [bulk_message_request_threshold](#bulk_message_request_threshold)
+  + [enable_ilm](#enable_ilm)
+  + [ilm_policy_id](#ilm_policy_id)
+  + [ilm_policy](#ilm_policy)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -97,6 +100,7 @@ Current maintainers: @cosmo0920
   + [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
   + [Fluentd seems to hang if it unable to connect Elasticsearch, why?](#fluentd-seems-to-hang-if-it-unable-to-connect-elasticsearch-why)
+  + [Enable Index Lifecycle Management](#enable-index-lifecycle-management)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -112,6 +116,8 @@ NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0
 
 NOTE: This documentation is for fluent-plugin-elasticsearch 2.x or later. For 1.x documentation, please see [v0.12 branch](https://github.com/uken/fluent-plugin-elasticsearch/tree/v0.12).
 
+NOTE: Using Index Lifecycle management(ILM) feature needs to install elasticsearch-xpack gem v7.4.0 or later.
+
 ## Installation
 
 ```sh
@@ -1120,6 +1126,30 @@ Default value is `20MB`. (20 * 1024 * 1024)
 
 If you specify this size as negative number, `bulk_message` request splitting feature will be disabled.
 
+## enable_ilm
+
+Enable Index Lifecycle Management (ILM).
+
+Default value is `false`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
+## ilm_policy_id
+
+Specify ILM policy id.
+
+Default value is `logstash-policy`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
+## ilm_policy
+
+Specify ILM policy contents as Hash.
+
+Default value is `{}`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch
@@ -1537,6 +1567,42 @@ To remove too pessimistic behavior, you can use the following configuration:
 </match>
 ```
 
+### Enable Index Lifecycle Management
+
+Index lifecycle management is template based index management feature.
+
+Main ILM feature parameters are:
+
+* `rollover_index`
+* `deflector_alias`
+* `enable_ilm`
+* `ilm_policy_id`
+* `ilm_policy`
+
+They are not all mandatory parameters but they are used for ILM feature in effect.
+
+And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
+Because ILM settings are injected into their Elasticsearch templates.
+
+```aconf
+index_name fluentd-${tag}
+# Should specify rollover_index as true
+rollover_index true
+deflector_alias fluentd-${tag} # Should specify as same index_name
+index_prefix fluentd
+application_name ${tag}
+index_date_pattern "now/d"
+enable_ilm true
+# Policy configurations
+ilm_policy_id fluentd-policy
+# ilm_policy {} # Use default policy
+template_name your-fluentd-template
+template_file /path/to/fluentd-template.json
+customize_template {"<<index_prefix>>": "fluentd"}
+```
+
+Note: This plugin only creates rollover-enabled indices, which are aliases pointing to them and index templates, and creates an ILM policy if enabled.
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.5.6'
+  s.version       = '3.6.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/gemfiles/Gemfile.ilm

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec :path => "../"
+
+gem 'simplecov', require: false
+gem 'coveralls', require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
+gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch-xpack"

data/lib/fluent/plugin/default-ilm-policy.json

@@ -0,0 +1,14 @@
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "50gb",
+            "max_age": "30d"
+          }
+        }
+      }
+    }
+  }
+}

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -0,0 +1,67 @@
+module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
+  ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
+
+  def setup_ilm(enable_ilm, policy_id, ilm_policy = default_policy_payload)
+    return unless enable_ilm
+
+    create_ilm_policy(policy_id, ilm_policy)
+  end
+
+  def verify_ilm_working
+    # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
+    # available and enabled.
+    begin
+      xpack = xpack_info
+      if xpack.nil?
+        raise Fluent::ConfigError, "xpack endpoint does not work"
+      end
+      features = xpack["features"]
+      ilm = features.nil? ? nil : features["ilm"]
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch" if features.nil? || ilm.nil?
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
+
+    rescue Elasticsearch::Transport::Transport::Error => e
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
+    end
+  end
+
+  def create_ilm_policy(policy_id, ilm_policy = default_policy_payload)
+    if !ilm_policy_exists?(policy_id)
+      ilm_policy_put(policy_id, ilm_policy)
+    end
+  end
+
+  def xpack_info
+    begin
+      client.xpack.info
+    rescue NoMethodError
+      raise RuntimeError, "elasticsearch-xpack gem is not installed."
+    rescue
+      nil
+    end
+  end
+
+  def get_ilm_policy
+    client.ilm.get_policy
+  end
+
+  def ilm_policy_exists?(policy_id)
+    begin
+      client.ilm.get_policy(policy_id: policy_id)
+      true
+    rescue
+      false
+    end
+  end
+
+  def ilm_policy_put(policy_id, policy)
+    log.info("Installing ILM policy: #{policy}")
+    client.ilm.put_policy(policy_id: policy_id, body: policy)
+  end
+
+  def default_policy_payload
+    default_policy_path = File.join(__dir__, ILM_DEFAULT_POLICY_PATH)
+    Yajl.load(::IO.read(default_policy_path))
+  end
+end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -21,8 +21,8 @@ module Fluent::ElasticsearchIndexTemplate
     JSON.parse(file_contents)
   end
 
-  def template_exists?(name)
-    client.indices.get_template(:name => name)
+  def template_exists?(name, host = nil)
+    client(host).indices.get_template(:name => name)
     return true
   rescue Elasticsearch::Transport::Transport::Errors::NotFound
     return false
@@ -51,52 +51,101 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_put(name, template)
-    client.indices.put_template(:name => name, :body => template)
+  def template_put(name, template, host = nil)
+    client(host).indices.put_template(:name => name, :body => template)
   end
 
-  def indexcreation(index_name)
-    client.indices.create(:index => index_name)
+  def indexcreation(index_name, host = nil)
+    client(host).indices.create(:index => index_name)
   rescue Elasticsearch::Transport::Transport::Error => e
+    if e.message =~ /"already exists"/
+      log.debug("Index #{index_name} already exists")
+    else
       log.error("Error while index creation - #{index_name}: #{e.inspect}")
+    end
   end
 
-  def template_install(name, template_file, overwrite)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil)
+    inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
-      template_put(name, get_template(template_file))
-      log.info("Template '#{name}' overwritten with #{template_file}.")
+      template_put(inject_template_name,
+                   enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                ilm_policy_id,
+                                                                get_template(template_file)) :
+                     get_template(template_file), host)
+
+      log.info("Template '#{inject_template_name}' overwritten with #{template_file}.")
       return
     end
-    if !template_exists?(name)
-      template_put(name, get_template(template_file))
-      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
+    if !template_exists?(inject_template_name, host)
+      template_put(inject_template_name,
+                   enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                ilm_policy_id,
+                                                                get_template(template_file)) :
+                     get_template(template_file), host)
+      log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
-      log.info("Template configured and already installed.")
+      log.debug("Template '#{inject_template_name}' configured and already installed.")
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern)
-    template_custom_name=template_name.downcase
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host)
+    template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
+    custom_template = if enable_ilm
+                        inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id,
+                                                        get_custom_template(template_file,
+                                                                            customize_template))
+                      else
+                        get_custom_template(template_file, customize_template)
+                      end
     if overwrite
-      template_put(template_custom_name, get_custom_template(template_file, customize_template))
+      template_put(template_custom_name, custom_template, host)
       log.info("Template '#{template_custom_name}' overwritten with #{template_file}.")
     else
-      if !template_exists?(template_custom_name)
-        template_put(template_custom_name, get_custom_template(template_file, customize_template))
+      if !template_exists?(template_custom_name, host)
+        template_put(template_custom_name, custom_template, host)
         log.info("Template configured, but no template installed. Installed '#{template_custom_name}' from #{template_file}.")
       else
-        log.info("Template configured and already installed.")
+        log.debug("Template '#{template_custom_name}' configured and already installed.")
       end
     end
+  end
+
+  def get_template_name(enable_ilm, template_name, deflector_alias_name)
+    enable_ilm ? deflector_alias_name : template_name
+  end
+
+  def inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id, template)
+    log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
+    template.delete('template') if template.include?('template')
+    template['index_patterns'] = "#{deflector_alias_name}-*"
+    template['order'] = template['order'] ? template['order'] + deflector_alias_name.split('-').length : 50 + deflector_alias_name.split('-').length
+    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    end
+    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias_name})
+    template
+  end
 
+  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
     if rollover_index
       if !client.indices.exists_alias(:name => deflector_alias_name)
-        index_name_temp='<'+index_prefix.downcase+'-'+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
-        indexcreation(index_name_temp)
-        client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name)
+        index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+        indexcreation(index_name_temp, host)
+        body = {}
+        body = rollover_alias_payload(deflector_alias_name) if enable_ilm
+        client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name,
+                                 :body => body)
         log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")
+        if enable_ilm
+          if ilm_policy.empty?
+            setup_ilm(enable_ilm, ilm_policy_id)
+          else
+            setup_ilm(enable_ilm, ilm_policy_id, ilm_policy)
+          end
+        end
       else
-        log.info("The alias '#{deflector_alias_name}' is already present")
+        log.debug("The alias '#{deflector_alias_name}' is already present")
       end
     else
       log.info("No index and alias creation action performed because rollover_index is set to '#{rollover_index}'")
@@ -109,4 +158,13 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
+  def rollover_alias_payload(rollover_alias)
+    {
+      'aliases' => {
+        rollover_alias => {
+          'is_write_index' =>  true
+        }
+      }
+    }
+  end
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,6 +2,10 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+begin
+  require 'elasticsearch/xpack'
+rescue LoadError
+end
 require 'json'
 require 'uri'
 begin
@@ -18,6 +22,7 @@ require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
+require_relative 'elasticsearch_index_lifecycle_management'
 begin
   require_relative 'oj_serializer'
 rescue LoadError
@@ -44,6 +49,8 @@ module Fluent::Plugin
 
     RequestInfo = Struct.new(:host, :index)
 
+    attr_reader :alias_indexes
+
     helpers :event_emitter, :compat_parameters, :record_accessor
 
     Fluent::Plugin.register_output('elasticsearch', self)
@@ -54,6 +61,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
     TARGET_BULK_BYTES = 20 * 1024 * 1024
+    DEFAULT_POLICY_ID = "logstash-policy"
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -105,6 +113,7 @@ EOC
     config_param :customize_template, :hash, :default => nil
     config_param :rollover_index, :string, :default => false
     config_param :index_date_pattern, :string, :default => "now/d"
+    config_param :index_separator, :string, :default => "-"
     config_param :deflector_alias, :string, :default => nil
     config_param :index_prefix, :string, :default => "logstash"
     config_param :application_name, :string, :default => "default"
@@ -139,6 +148,9 @@ EOC
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
     config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :enable_ilm, :bool, :default => false
+    config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
+    config_param :ilm_policy, :hash, :default => {}
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -148,6 +160,7 @@ EOC
 
     include Fluent::ElasticsearchIndexTemplate
     include Fluent::Plugin::ElasticsearchConstants
+    include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
     def initialize
       super
@@ -181,24 +194,25 @@ EOC
       raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
       raise Fluent::ConfigError, "'max_retry_get_es_version' must be greater than or equal to zero." if @max_retry_get_es_version < 0
 
-      # Raise error when using host placeholders and template features at same time.
+      # Dump log when using host placeholders and template features at same time.
       valid_host_placeholder = placeholder?(:host_placeholder, @host)
       if valid_host_placeholder && (@template_name && @template_file || @templates)
-        raise Fluent::ConfigError, "host placeholder and template installation are exclusive features."
+        if @verify_es_version_at_startup
+          raise Fluent::ConfigError, "host placeholder, template installation, and verify Elasticsearch version at startup are exclusive feature at same time. Please specify verify_es_version_at_startup as `false` when host placeholder and template installation are enabled."
+        end
+        log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
       end
 
+      @alias_indexes = []
       if !Fluent::Engine.dry_run_mode
         if @template_name && @template_file
-          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
-            if @customize_template
-              if @rollover_index
-                raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
-              end
-              template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix, @rollover_index, @deflector_alias, @application_name, @index_date_pattern)
-            else
-              template_install(@template_name, @template_file, @template_overwrite)
-            end
+          if @rollover_index
+            raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
           end
+          if @enable_ilm
+            raise Fluent::ConfigError, "'rollover_index' and 'deflector_alias' must be provided if 'enable_ilm' is set true ." if !@deflector_alias &&!@deflector_alias
+          end
+          verify_ilm_working if @enable_ilm
         elsif @templates
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
@@ -579,7 +593,17 @@ EOC
       else
         type_name = nil
       end
-      return logstash_prefix, index_name, type_name
+      if @deflector_alias
+        deflector_alias = extract_placeholders(@deflector_alias, chunk)
+      else
+        deflector_alias = nil
+      end
+      if @application_name
+        application_name = extract_placeholders(@application_name, chunk)
+      else
+        application_name = nil
+      end
+      return logstash_prefix, index_name, type_name, deflector_alias, application_name
     end
 
     def multi_workers_ready?
@@ -653,7 +677,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, index_name, type_name = extracted_values
+      logstash_prefix, index_name, type_name, deflector_alias, application_name = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -747,6 +771,23 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
+      logstash_prefix, index_name, type_name, deflector_alias, application_name = extracted_values
+      if @template_name && @template_file
+        if @alias_indexes.include? deflector_alias
+          log.debug("Index alias #{deflector_alias} already exists (cached)")
+        else
+          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
+            if @customize_template
+              template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, info.host)
+            else
+              template_install(@template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, info.host)
+            end
+            create_rollover_alias(@index_prefix, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, info.host)
+          end
+          @alias_indexes << deflector_alias unless deflector_alias.nil?
+        end
+      end
+
       begin
 
         log.on_trace { log.trace "bulk request: #{data}" }