fluent-plugin-elasticsearch 3.5.4 → 3.5.5

data/Rakefile

@@ -1,11 +1,11 @@
-require 'bundler/gem_tasks'
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.warning = false
-end
-
-task :default => :test
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/appveyor.yml

@@ -1,30 +1,30 @@
-version: '{build}'
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - "%devkit%\\devkitvars.bat"
-  - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
-  - ruby --version
-  - gem --version
-  - bundle install
-build: off
-test_script:
-  - bundle exec rake test
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "25-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "25"
-      devkit: C:\Ruby23\DevKit
-    - ruby_version: "24-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "24"
-      devkit: C:\Ruby23\DevKit
-    - ruby_version: "23-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "22-x64"
-      devkit: C:\Ruby23-x64\DevKit
-matrix:
-  allow_failures:
-    - ruby_version: "21"
+version: '{build}'
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - "%devkit%\\devkitvars.bat"
+  - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "25-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "25"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "24-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "24"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "23-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "22-x64"
+      devkit: C:\Ruby23-x64\DevKit
+matrix:
+  allow_failures:
+    - ruby_version: "21"

data/fluent-plugin-elasticsearch.gemspec

@@ -1,31 +1,31 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path('../lib', __FILE__)
-
-Gem::Specification.new do |s|
-  s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.5.4'
-  s.authors       = ['diogo', 'pitr']
-  s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
-  s.description   = %q{Elasticsearch output plugin for Fluent event collector}
-  s.summary       = s.description
-  s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
-  s.license       = 'Apache-2.0'
-
-  s.files         = `git ls-files`.split($/)
-  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-  s.require_paths = ['lib']
-
-  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
-
-  s.add_runtime_dependency 'fluentd', '>= 0.14.22'
-  s.add_runtime_dependency 'excon', '>= 0'
-  s.add_runtime_dependency 'elasticsearch'
-
-
-  s.add_development_dependency 'rake', '>= 0'
-  s.add_development_dependency 'webmock', '~> 1'
-  s.add_development_dependency 'test-unit', '~> 3.1.0'
-  s.add_development_dependency 'minitest', '~> 5.8'
-  s.add_development_dependency 'flexmock', '~> 2.0'
-end
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'fluent-plugin-elasticsearch'
+  s.version       = '3.5.5'
+  s.authors       = ['diogo', 'pitr']
+  s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
+  s.description   = %q{Elasticsearch output plugin for Fluent event collector}
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
+  s.license       = 'Apache-2.0'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
+
+  s.add_runtime_dependency 'fluentd', '>= 0.14.22'
+  s.add_runtime_dependency 'excon', '>= 0'
+  s.add_runtime_dependency 'elasticsearch'
+
+
+  s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webmock', '~> 1'
+  s.add_development_dependency 'test-unit', '~> 3.1.0'
+  s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'flexmock', '~> 2.0'
+end

data/lib/fluent/log-ext.rb

@@ -1,38 +1,38 @@
-require 'fluent/log'
-# For elasticsearch-ruby v7.0.0 or later
-# logger for Elasticsearch::Loggable required the following methods:
-#
-# * debug?
-# * info?
-# * warn?
-# * error?
-# * fatal?
-
-module Fluent
-  class Log
-    # Elasticsearch::Loggable does not request trace? method.
-    # def trace?
-    #   @level <= LEVEL_TRACE
-    # end
-
-    def debug?
-      @level <= LEVEL_DEBUG
-    end
-
-    def info?
-      @level <= LEVEL_INFO
-    end
-
-    def warn?
-      @level <= LEVEL_WARN
-    end
-
-    def error?
-      @level <= LEVEL_ERROR
-    end
-
-    def fatal?
-      @level <= LEVEL_FATAL
-    end
-  end
-end
+require 'fluent/log'
+# For elasticsearch-ruby v7.0.0 or later
+# logger for Elasticsearch::Loggable required the following methods:
+#
+# * debug?
+# * info?
+# * warn?
+# * error?
+# * fatal?
+
+module Fluent
+  class Log
+    # Elasticsearch::Loggable does not request trace? method.
+    # def trace?
+    #   @level <= LEVEL_TRACE
+    # end
+
+    def debug?
+      @level <= LEVEL_DEBUG
+    end
+
+    def info?
+      @level <= LEVEL_INFO
+    end
+
+    def warn?
+      @level <= LEVEL_WARN
+    end
+
+    def error?
+      @level <= LEVEL_ERROR
+    end
+
+    def fatal?
+      @level <= LEVEL_FATAL
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -1,13 +1,13 @@
-module Fluent
-  module Plugin
-    module ElasticsearchConstants
-      BODY_DELIMITER = "\n".freeze
-      UPDATE_OP = "update".freeze
-      UPSERT_OP = "upsert".freeze
-      CREATE_OP = "create".freeze
-      INDEX_OP = "index".freeze
-      ID_FIELD = "_id".freeze
-      TIMESTAMP_FIELD = "@timestamp".freeze
-    end
-  end
-end
+module Fluent
+  module Plugin
+    module ElasticsearchConstants
+      BODY_DELIMITER = "\n".freeze
+      UPDATE_OP = "update".freeze
+      UPSERT_OP = "upsert".freeze
+      CREATE_OP = "create".freeze
+      INDEX_OP = "index".freeze
+      ID_FIELD = "_id".freeze
+      TIMESTAMP_FIELD = "@timestamp".freeze
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_error.rb

@@ -1,5 +1,5 @@
-require 'fluent/error'
-
-class Fluent::Plugin::ElasticsearchError
-  class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
-end
+require 'fluent/error'
+
+class Fluent::Plugin::ElasticsearchError
+  class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -1,122 +1,127 @@
-require 'fluent/event'
-require 'fluent/error'
-require_relative 'elasticsearch_constants'
-
-class Fluent::Plugin::ElasticsearchErrorHandler
-  include Fluent::Plugin::ElasticsearchConstants
-
-  attr_accessor :bulk_message_count
-  class ElasticsearchVersionMismatch < Fluent::UnrecoverableError; end
-  class ElasticsearchSubmitMismatch < Fluent::UnrecoverableError; end
-  class ElasticsearchRequestAbortError < Fluent::UnrecoverableError; end
-  class ElasticsearchError < StandardError; end
-
-  def initialize(plugin)
-    @plugin = plugin
-  end
-
-  def unrecoverable_error_types
-    @plugin.unrecoverable_error_types
-  end
-
-  def unrecoverable_error?(type)
-    unrecoverable_error_types.include?(type)
-  end
-
-  def log_es_400_reason(&block)
-    if @plugin.log_es_400_reason
-      block.call
-    else
-      @plugin.log.on_debug &block
-    end
-  end
-
-  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
-    items = response['items']
-    if items.nil? || !items.is_a?(Array)
-      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
-    end
-    if bulk_message_count != items.length
-      raise ElasticsearchSubmitMismatch, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
-    end
-    retry_stream = Fluent::MultiEventStream.new
-    stats = Hash.new(0)
-    meta = {}
-    header = {}
-    chunk.msgpack_each do |time, rawrecord|
-      bulk_message = ''
-      next unless rawrecord.is_a? Hash
-      begin
-        # we need a deep copy for process_message to alter
-        processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
-        next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
-      rescue => e
-        stats[:bad_chunk_record] += 1
-        next
-      end
-      item = items.shift
-      if item.is_a?(Hash) && item.has_key?(@plugin.write_operation)
-        write_operation = @plugin.write_operation
-      elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
-        write_operation = CREATE_OP
-      elsif item.nil?
-        stats[:errors_nil_resp] += 1
-        next
-      else
-        # When we don't have an expected ops field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      if item[write_operation].has_key?('status')
-        status = item[write_operation]['status']
-      else
-        # When we don't have a status field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      case
-      when [200, 201].include?(status)
-        stats[:successes] += 1
-      when CREATE_OP == write_operation && 409 == status
-        stats[:duplicates] += 1
-      when 400 == status
-        stats[:bad_argument] += 1
-        reason = ""
-        log_es_400_reason do
-          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
-            reason = " [error type]: #{item[write_operation]['error']['type']}"
-          end
-          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
-            reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
-          end
-        end
-        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("400 - Rejected by Elasticsearch#{reason}"))
-      else
-        if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
-          type = item[write_operation]['error']['type']
-          stats[type] += 1
-          retry_stream.add(time, rawrecord)
-          if unrecoverable_error?(type)
-            raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
-          end
-        else
-          # When we don't have a type field, something changed in the API
-          # expected return values (ES 2.x)
-          stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
-          next
-        end
-        stats[type] += 1
-      end
-    end
-    @plugin.log.on_debug do
-      msg = ["Indexed (op = #{@plugin.write_operation})"]
-      stats.each_pair { |key, value| msg << "#{value} #{key}" }
-      @plugin.log.debug msg.join(', ')
-    end
-    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
-  end
-end
+require 'fluent/event'
+require 'fluent/error'
+require_relative 'elasticsearch_constants'
+
+class Fluent::Plugin::ElasticsearchErrorHandler
+  include Fluent::Plugin::ElasticsearchConstants
+
+  attr_accessor :bulk_message_count
+  class ElasticsearchVersionMismatch < Fluent::UnrecoverableError; end
+  class ElasticsearchSubmitMismatch < Fluent::UnrecoverableError; end
+  class ElasticsearchRequestAbortError < Fluent::UnrecoverableError; end
+  class ElasticsearchError < StandardError; end
+
+  def initialize(plugin)
+    @plugin = plugin
+  end
+
+  def unrecoverable_error_types
+    @plugin.unrecoverable_error_types
+  end
+
+  def unrecoverable_error?(type)
+    unrecoverable_error_types.include?(type)
+  end
+
+  def log_es_400_reason(&block)
+    if @plugin.log_es_400_reason
+      block.call
+    else
+      @plugin.log.on_debug &block
+    end
+  end
+
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array)
+      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+      raise ElasticsearchSubmitMismatch, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
+    stats = Hash.new(0)
+    meta = {}
+    header = {}
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
+      rescue => e
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
+      if item.is_a?(Hash) && item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      elsif item.nil?
+        stats[:errors_nil_resp] += 1
+        next
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
+      when CREATE_OP == write_operation && 409 == status
+        stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        reason = ""
+        log_es_400_reason do
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+            reason = " [error type]: #{item[write_operation]['error']['type']}"
+          end
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
+            reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
+          end
+        end
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("400 - Rejected by Elasticsearch#{reason}"))
+      else
+        if item[write_operation]['error'].is_a?(String)
+          reason = item[write_operation]['error']
+          stats[:errors_block_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{reason}"))
+          next
+        elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+          stats[type] += 1
+          retry_stream.add(time, rawrecord)
+          if unrecoverable_error?(type)
+            raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
+          end
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values (ES 2.x)
+          stats[:errors_bad_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
+          next
+        end
+        stats[type] += 1
+      end
+    end
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
+    end
+    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
+  end
+end