fluent-plugin-elasticsearch-dext 5.0.2

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/appveyor.yml

@@ -0,0 +1,20 @@
+version: '{build}'
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - ridk.cmd enable
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "26-x64"
+    - ruby_version: "26"
+    - ruby_version: "25-x64"
+    - ruby_version: "25"
+    - ruby_version: "24-x64"
+    - ruby_version: "24"

data/fluent-plugin-elasticsearch.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'fluent-plugin-elasticsearch-dext'
+  s.version       = '5.0.2'
+  s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake', 'mihailgmihaylov']
+  s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com', 'mihail.mihaylov@dext.com']
+  s.description   = %q{Elasticsearch output plugin for Fluent event collector with small modification from Dext}
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/wbrefvem/fluent-plugin-elasticsearch'
+  s.license       = 'Apache-2.0'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  if s.respond_to?(:metadata)
+    s.metadata["changelog_uri"] = "https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md"
+  end
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
+
+  s.add_runtime_dependency 'fluentd', '>= 0.14.22'
+  s.add_runtime_dependency 'excon', '>= 0'
+  s.add_runtime_dependency 'elasticsearch'
+
+
+  s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webmock', '~> 3'
+  s.add_development_dependency 'test-unit', '~> 3.3.0'
+  s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'flexmock', '~> 2.0'
+end

data/gemfiles/Gemfile.elasticsearch.v6

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec :path => "../"
+
+gem 'simplecov', require: false
+gem 'coveralls', ">= 0.8.0", require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
+gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch", "~> 6.8.1"
+gem "elasticsearch-xpack"
+gem "oj"

data/lib/fluent/log-ext.rb

@@ -0,0 +1,38 @@
+require 'fluent/log'
+# For elasticsearch-ruby v7.0.0 or later
+# logger for Elasticsearch::Loggable required the following methods:
+#
+# * debug?
+# * info?
+# * warn?
+# * error?
+# * fatal?
+
+module Fluent
+  class Log
+    # Elasticsearch::Loggable does not request trace? method.
+    # def trace?
+    #   @level <= LEVEL_TRACE
+    # end
+
+    def debug?
+      @level <= LEVEL_DEBUG
+    end
+
+    def info?
+      @level <= LEVEL_INFO
+    end
+
+    def warn?
+      @level <= LEVEL_WARN
+    end
+
+    def error?
+      @level <= LEVEL_ERROR
+    end
+
+    def fatal?
+      @level <= LEVEL_FATAL
+    end
+  end
+end

data/lib/fluent/plugin/default-ilm-policy.json

@@ -0,0 +1,14 @@
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "50gb",
+            "max_age": "30d"
+          }
+        }
+      }
+    }
+  }
+}

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -0,0 +1,13 @@
+module Fluent
+  module Plugin
+    module ElasticsearchConstants
+      BODY_DELIMITER = "\n".freeze
+      UPDATE_OP = "update".freeze
+      UPSERT_OP = "upsert".freeze
+      CREATE_OP = "create".freeze
+      INDEX_OP = "index".freeze
+      ID_FIELD = "_id".freeze
+      TIMESTAMP_FIELD = "@timestamp".freeze
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_error.rb

@@ -0,0 +1,5 @@
+require 'fluent/error'
+
+class Fluent::Plugin::ElasticsearchError
+  class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -0,0 +1,129 @@
+require 'fluent/event'
+require 'fluent/error'
+require_relative 'elasticsearch_constants'
+
+class Fluent::Plugin::ElasticsearchErrorHandler
+  include Fluent::Plugin::ElasticsearchConstants
+
+  attr_accessor :bulk_message_count
+  class ElasticsearchVersionMismatch < Fluent::UnrecoverableError; end
+  class ElasticsearchSubmitMismatch < Fluent::UnrecoverableError; end
+  class ElasticsearchRequestAbortError < Fluent::UnrecoverableError; end
+  class ElasticsearchError < StandardError; end
+
+  def initialize(plugin)
+    @plugin = plugin
+  end
+
+  def unrecoverable_error_types
+    @plugin.unrecoverable_error_types
+  end
+
+  def unrecoverable_error?(type)
+    unrecoverable_error_types.include?(type)
+  end
+
+  def log_es_400_reason(&block)
+    if @plugin.log_es_400_reason
+      block.call
+    else
+      @plugin.log.on_debug(&block)
+    end
+  end
+
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array)
+      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+      raise ElasticsearchSubmitMismatch, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
+    stats = Hash.new(0)
+    meta = {}
+    header = {}
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
+      rescue => e
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
+      if item.is_a?(Hash) && item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      elsif UPSERT_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(UPDATE_OP)
+        write_operation = UPDATE_OP
+      elsif item.nil?
+        stats[:errors_nil_resp] += 1
+        next
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
+      when CREATE_OP == write_operation && 409 == status
+        stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        reason = ""
+        log_es_400_reason do
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+            reason = " [error type]: #{item[write_operation]['error']['type']}"
+          end
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
+            reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
+          end
+        end
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("400 - Rejected by Elasticsearch#{reason}"))
+      else
+        if item[write_operation]['error'].is_a?(String)
+          reason = item[write_operation]['error']
+          stats[:errors_block_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{reason}"))
+          next
+        elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+          stats[type] += 1
+          retry_stream.add(time, rawrecord)
+          if unrecoverable_error?(type)
+            raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
+          end
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values (ES 2.x)
+          stats[:errors_bad_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
+          next
+        end
+        stats[type] += 1
+      end
+    end
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
+    end
+    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
+  end
+end

data/lib/fluent/plugin/elasticsearch_fallback_selector.rb

@@ -0,0 +1,9 @@
+require 'elasticsearch/transport/transport/connections/selector'
+
+class Fluent::Plugin::ElasticseatchFallbackSelector
+  include Elasticsearch::Transport::Transport::Connections::Selector::Base
+
+  def select(options={})
+    connections.first
+  end
+end

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -0,0 +1,67 @@
+module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
+  ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
+
+  def setup_ilm(enable_ilm, policy_id, ilm_policy = default_policy_payload, overwrite = false)
+    return unless enable_ilm
+
+    create_ilm_policy(policy_id, ilm_policy, overwrite)
+  end
+
+  def verify_ilm_working
+    # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
+    # available and enabled.
+    begin
+      xpack = xpack_info
+      if xpack.nil?
+        raise Fluent::ConfigError, "xpack endpoint does not work"
+      end
+      features = xpack["features"]
+      ilm = features.nil? ? nil : features["ilm"]
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch" if features.nil? || ilm.nil?
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
+
+    rescue Elasticsearch::Transport::Transport::Error => e
+      raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
+    end
+  end
+
+  def create_ilm_policy(policy_id, ilm_policy = default_policy_payload, overwrite = false)
+    if overwrite || !ilm_policy_exists?(policy_id)
+      ilm_policy_put(policy_id, ilm_policy)
+    end
+  end
+
+  def xpack_info
+    begin
+      client.xpack.info
+    rescue NoMethodError
+      raise RuntimeError, "elasticsearch-xpack gem is not installed."
+    rescue
+      nil
+    end
+  end
+
+  def get_ilm_policy
+    client.ilm.get_policy
+  end
+
+  def ilm_policy_exists?(policy_id)
+    begin
+      client.ilm.get_policy(policy_id: policy_id)
+      true
+    rescue
+      false
+    end
+  end
+
+  def ilm_policy_put(policy_id, policy)
+    log.info("Installing ILM policy: #{policy}")
+    client.ilm.put_policy(policy_id: policy_id, body: policy)
+  end
+
+  def default_policy_payload
+    default_policy_path = File.join(__dir__, ILM_DEFAULT_POLICY_PATH)
+    Yajl.load(::IO.read(default_policy_path))
+  end
+end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -0,0 +1,211 @@
+require 'fluent/error'
+require_relative './elasticsearch_error'
+
+module Fluent::ElasticsearchIndexTemplate
+  def get_template(template_file)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    JSON.parse(file_contents)
+  end
+
+  def get_custom_template(template_file, customize_template)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    customize_template.each do |key, value|
+      file_contents = file_contents.gsub(key,value.downcase)
+    end
+    JSON.parse(file_contents)
+  end
+
+  def template_exists?(name, host = nil)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
+    return true
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    return false
+  end
+
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
+    return unless block_given?
+    retries = 0
+    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
+    begin
+      yield
+    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+      @_es = nil
+      @_es_info = nil
+      if retries < max_retries
+        retries += 1
+        wait_seconds = 2**retries
+        sleep wait_seconds
+        log.warn "Could not communicate to Elasticsearch, resetting connection and trying again. #{e.message}"
+        log.warn "Remaining retry: #{max_retries - retries}. Retry to communicate after #{wait_seconds} second(s)."
+        retry
+      end
+      message = "Could not communicate to Elasticsearch after #{retries} retries. #{e.message}"
+      log.warn message
+      raise Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure,
+            message if fail_on_retry_exceed
+    end
+  end
+
+  def template_put(name, template, host = nil)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
+  end
+
+  def indexcreation(index_name, host = nil)
+    client(host).indices.create(:index => index_name)
+  rescue Elasticsearch::Transport::Transport::Error => e
+    if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
+      log.debug("Index #{index_name} already exists")
+    else
+      log.error("Error while index creation - #{index_name}", error: e)
+    end
+  end
+
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
+    inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
+    if overwrite
+      template_put(inject_template_name,
+                   enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
+                                                                ilm_policy_id,
+                                                                get_template(template_file),
+                                                                index_separator) :
+                     get_template(template_file), host)
+
+      log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
+      return
+    end
+    if !template_exists?(inject_template_name, host)
+      template_put(inject_template_name,
+                   enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
+                                                                ilm_policy_id,
+                                                                get_template(template_file),
+                                                                index_separator) :
+                     get_template(template_file), host)
+      log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
+    else
+      log.debug("Template '#{inject_template_name}' configured and already installed.")
+    end
+  end
+
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
+    template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
+    custom_template = if enable_ilm
+                        inject_ilm_settings_to_template(deflector_alias_name,
+                                                        target_index,
+                                                        ilm_policy_id,
+                                                        get_custom_template(template_file,
+                                                                            customize_template),
+                                                        index_separator)
+                      else
+                        get_custom_template(template_file, customize_template)
+                      end
+    if overwrite
+      template_put(template_custom_name, custom_template, host)
+      log.info("Template '#{template_custom_name}' overwritten with #{template_file}.")
+    else
+      if !template_exists?(template_custom_name, host)
+        template_put(template_custom_name, custom_template, host)
+        log.info("Template configured, but no template installed. Installed '#{template_custom_name}' from #{template_file}.")
+      else
+        log.debug("Template '#{template_custom_name}' configured and already installed.")
+      end
+    end
+  end
+
+  def get_template_name(enable_ilm, template_name, deflector_alias_name)
+    enable_ilm ? deflector_alias_name : template_name
+  end
+
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
+    log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
+    end
+    template
+  end
+
+  def create_rollover_alias(target_index, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
+     # ILM request to create alias.
+    if rollover_index || enable_ilm
+      if !client.indices.exists_alias(:name => deflector_alias_name)
+        if @logstash_format
+          index_name_temp = '<'+target_index+'-000001>'
+        else
+          if index_date_pattern.empty?
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-000001>'
+          else
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+          end
+        end
+        indexcreation(index_name_temp, host)
+        body = rollover_alias_payload(deflector_alias_name)
+        client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name,
+                                 :body => body)
+        log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")
+      else
+        log.debug("The alias '#{deflector_alias_name}' is already present")
+      end
+      # Create ILM policy if rollover indices exist.
+      if enable_ilm
+        if ilm_policy.empty?
+          setup_ilm(enable_ilm, ilm_policy_id)
+        else
+          setup_ilm(enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite)
+        end
+      end
+    else
+      log.debug("No index and alias creation action performed because rollover_index or enable_ilm is set to: '#{rollover_index}', '#{enable_ilm}'")
+    end
+  end
+
+  def templates_hash_install(templates, overwrite)
+    templates.each do |key, value|
+      template_install(key, value, overwrite)
+    end
+  end
+
+  def rollover_alias_payload(rollover_alias)
+    {
+      'aliases' => {
+        rollover_alias => {
+          'is_write_index' =>  true
+        }
+      }
+    }
+  end
+end