fluent-plugin-elasticsearch 2.10.2 → 2.10.3

data/Rakefile

@@ -1,11 +1,11 @@
-require 'bundler/gem_tasks'
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.warning = false
-end
-
-task :default => :test
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/appveyor.yml

@@ -1,35 +1,35 @@
-version: '{build}'
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - "%devkit%\\devkitvars.bat"
-  - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
-  - ruby --version
-  - gem --version
-  - bundle install
-build: off
-test_script:
-  - bundle exec rake test
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "25-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "25"
-      devkit: C:\Ruby23\DevKit
-    - ruby_version: "24-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "24"
-      devkit: C:\Ruby23\DevKit
-    - ruby_version: "23-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "22-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "21-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "21"
-      devkit: C:\Ruby23\DevKit
-      WIN_RAPID: true
-matrix:
-  allow_failures:
-    - ruby_version: "21"
+version: '{build}'
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - "%devkit%\\devkitvars.bat"
+  - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
+  - ruby --version
+  - gem --version
+  - bundle install
+build: off
+test_script:
+  - bundle exec rake test
+
+# https://www.appveyor.com/docs/installed-software/#ruby
+environment:
+  matrix:
+    - ruby_version: "25-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "25"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "24-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "24"
+      devkit: C:\Ruby23\DevKit
+    - ruby_version: "23-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "22-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21"
+      devkit: C:\Ruby23\DevKit
+      WIN_RAPID: true
+matrix:
+  allow_failures:
+    - ruby_version: "21"

data/fluent-plugin-elasticsearch.gemspec

@@ -1,31 +1,31 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path('../lib', __FILE__)
-
-Gem::Specification.new do |s|
-  s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.10.2'
-  s.authors       = ['diogo', 'pitr']
-  s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
-  s.description   = %q{Elasticsearch output plugin for Fluent event collector}
-  s.summary       = s.description
-  s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
-  s.license       = 'Apache-2.0'
-
-  s.files         = `git ls-files`.split($/)
-  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-  s.require_paths = ['lib']
-
-  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
-
-  s.add_runtime_dependency 'fluentd', '>= 0.14.20'
-  s.add_runtime_dependency 'excon', '>= 0'
-  s.add_runtime_dependency 'elasticsearch'
-
-
-  s.add_development_dependency 'rake', '>= 0'
-  s.add_development_dependency 'webmock', '~> 1'
-  s.add_development_dependency 'test-unit', '~> 3.1.0'
-  s.add_development_dependency 'minitest', '~> 5.8'
-  s.add_development_dependency 'flexmock', '~> 2.0'
-end
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'fluent-plugin-elasticsearch'
+  s.version       = '2.10.3'
+  s.authors       = ['diogo', 'pitr']
+  s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
+  s.description   = %q{Elasticsearch output plugin for Fluent event collector}
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
+  s.license       = 'Apache-2.0'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
+
+  s.add_runtime_dependency 'fluentd', '>= 0.14.20'
+  s.add_runtime_dependency 'excon', '>= 0'
+  s.add_runtime_dependency 'elasticsearch'
+
+
+  s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webmock', '~> 1'
+  s.add_development_dependency 'test-unit', '~> 3.1.0'
+  s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'flexmock', '~> 2.0'
+end

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -1,13 +1,13 @@
-module Fluent
-  module Plugin
-    module ElasticsearchConstants
-      BODY_DELIMITER = "\n".freeze
-      UPDATE_OP = "update".freeze
-      UPSERT_OP = "upsert".freeze
-      CREATE_OP = "create".freeze
-      INDEX_OP = "index".freeze
-      ID_FIELD = "_id".freeze
-      TIMESTAMP_FIELD = "@timestamp".freeze
-    end
-  end
-end
+module Fluent
+  module Plugin
+    module ElasticsearchConstants
+      BODY_DELIMITER = "\n".freeze
+      UPDATE_OP = "update".freeze
+      UPSERT_OP = "upsert".freeze
+      CREATE_OP = "create".freeze
+      INDEX_OP = "index".freeze
+      ID_FIELD = "_id".freeze
+      TIMESTAMP_FIELD = "@timestamp".freeze
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -1,87 +1,87 @@
-require 'fluent/event'
-require_relative 'elasticsearch_constants'
-
-class Fluent::Plugin::ElasticsearchErrorHandler
-  include Fluent::Plugin::ElasticsearchConstants
-
-  attr_accessor :bulk_message_count
-  class ElasticsearchVersionMismatch < StandardError; end
-  class ElasticsearchError < StandardError; end
-
-  def initialize(plugin)
-    @plugin = plugin
-  end
-
-  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
-    items = response['items']
-    if items.nil? || !items.is_a?(Array)
-      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
-    end
-    if bulk_message_count != items.length
-        raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
-    end
-    retry_stream = Fluent::MultiEventStream.new
-    stats = Hash.new(0)
-    meta = {}
-    header = {}
-    chunk.msgpack_each do |time, rawrecord|
-      bulk_message = ''
-      next unless rawrecord.is_a? Hash
-      begin
-        # we need a deep copy for process_message to alter
-        processrecord = Marshal.load(Marshal.dump(rawrecord))
-        @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
-      rescue => e
-        stats[:bad_chunk_record] += 1
-        next
-      end
-      item = items.shift
-      if item.has_key?(@plugin.write_operation)
-        write_operation = @plugin.write_operation
-      elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
-        write_operation = CREATE_OP
-      else
-        # When we don't have an expected ops field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      if item[write_operation].has_key?('status')
-        status = item[write_operation]['status']
-      else
-        # When we don't have a status field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      case
-      when [200, 201].include?(status)
-        stats[:successes] += 1
-      when CREATE_OP == write_operation && 409 == status
-        stats[:duplicates] += 1
-      when 400 == status
-        stats[:bad_argument] += 1
-        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
-      else
-        if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
-          type = item[write_operation]['error']['type']
-          stats[type] += 1
-          retry_stream.add(time, rawrecord)
-        else
-          # When we don't have a type field, something changed in the API
-          # expected return values (ES 2.x)
-          stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
-          next
-        end
-        stats[type] += 1
-      end
-    end
-    @plugin.log.on_debug do
-      msg = ["Indexed (op = #{@plugin.write_operation})"]
-      stats.each_pair { |key, value| msg << "#{value} #{key}" }
-      @plugin.log.debug msg.join(', ')
-    end
-    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
-  end
-end
+require 'fluent/event'
+require_relative 'elasticsearch_constants'
+
+class Fluent::Plugin::ElasticsearchErrorHandler
+  include Fluent::Plugin::ElasticsearchConstants
+
+  attr_accessor :bulk_message_count
+  class ElasticsearchVersionMismatch < StandardError; end
+  class ElasticsearchError < StandardError; end
+
+  def initialize(plugin)
+    @plugin = plugin
+  end
+
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array)
+      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+        raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
+    stats = Hash.new(0)
+    meta = {}
+    header = {}
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
+      rescue => e
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
+      if item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values (ES 2.x)
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
+      when CREATE_OP == write_operation && 409 == status
+        stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
+      else
+        if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+          stats[type] += 1
+          retry_stream.add(time, rawrecord)
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values (ES 2.x)
+          stats[:errors_bad_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
+          next
+        end
+        stats[type] += 1
+      end
+    end
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
+    end
+    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
+  end
+end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -1,42 +1,60 @@
-module Fluent::ElasticsearchIndexTemplate
-
-  def get_template(template_file)
-    if !File.exists?(template_file)
-      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
-    end
-    file_contents = IO.read(template_file).gsub(/\n/,'')
-    JSON.parse(file_contents)
-  end
-
-  def template_exists?(name)
-    client.indices.get_template(:name => name)
-    return true
-  rescue Elasticsearch::Transport::Transport::Errors::NotFound
-    return false
-  end
-
-  def template_put(name, template)
-    client.indices.put_template(:name => name, :body => template)
-  end
-
-  def template_install(name, template_file, overwrite)
-    if overwrite
-      template_put(name, get_template(template_file))
-      log.info("Template '#{name}' overwritten with #{template_file}.")
-      return
-    end
-    if !template_exists?(name)
-      template_put(name, get_template(template_file))
-      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
-    else
-      log.info("Template configured and already installed.")
-    end
-  end
-
-  def templates_hash_install(templates, overwrite)
-    templates.each do |key, value|
-      template_install(key, value, overwrite)
-    end
-  end
-
-end
+module Fluent::ElasticsearchIndexTemplate
+
+  def get_template(template_file)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    JSON.parse(file_contents)
+  end
+
+  def template_exists?(name)
+    client.indices.get_template(:name => name)
+    return true
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    return false
+  end
+
+  def retry_install(max_retries)
+    return unless block_given?
+    retries = 0
+    begin
+      yield
+    rescue Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, Timeout::Error => e
+      @_es = nil
+      @_es_info = nil
+      if retries < max_retries
+        retries += 1
+        sleep 2**retries
+        log.warn "Could not push template(s) to Elasticsearch, resetting connection and trying again. #{e.message}"
+        retry
+      end
+      raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Could not push template(s) to Elasticsearch after #{retries} retries. #{e.message}"
+    end
+  end
+
+  def template_put(name, template)
+    client.indices.put_template(:name => name, :body => template)
+  end
+
+  def template_install(name, template_file, overwrite)
+    if overwrite
+      template_put(name, get_template(template_file))
+      log.info("Template '#{name}' overwritten with #{template_file}.")
+      return
+    end
+    if !template_exists?(name)
+      template_put(name, get_template(template_file))
+      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
+    else
+      log.info("Template configured and already installed.")
+    end
+  end
+
+  def templates_hash_install(templates, overwrite)
+    templates.each do |key, value|
+      template_install(key, value, overwrite)
+    end
+  end
+
+end