fluent-plugin-elasticsearch 1.18.2 → 2.0.0.rc.1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.18.2
+  version: 2.0.0.rc.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-17 00:00:00.000000000 Z
+date: 2017-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.10
+        version: 0.14.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.10
+        version: 0.14.8
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
@@ -115,15 +115,15 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.5
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.5
-description: Elasticsearch output plugin for Fluent event collector
+        version: '2.0'
+description: ElasticSearch output plugin for Fluent event collector
 email:
 - pitr.vern@gmail.com
 - me@diogoterror.com
@@ -136,7 +136,6 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- Gemfile.v0.12
 - History.md
 - ISSUE_TEMPLATE.md
 - LICENSE.txt
@@ -144,24 +143,16 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
-- lib/fluent/log-ext.rb
-- lib/fluent/plugin/elasticsearch_constants.rb
-- lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
-- lib/fluent/plugin/elasticsearch_simple_sniffer.rb
-- lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
-- test/plugin/test_elasticsearch_error_handler.rb
-- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
-- test/test_log-ext.rb
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
-- Apache-2.0
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -174,19 +165,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Elasticsearch output plugin for Fluent event collector
+summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
-- test/plugin/test_elasticsearch_error_handler.rb
-- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
-- test/test_log-ext.rb

data/Gemfile.v0.12

@@ -1,11 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
-gem 'fluentd', '~> 0.12.0'
-
-gemspec
-
-
-gem 'simplecov', require: false
-gem 'coveralls', require: false
-gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/

data/lib/fluent/log-ext.rb

@@ -1,38 +0,0 @@
-require 'fluent/log'
-# For elasticsearch-ruby v7.0.0 or later
-# logger for Elasticsearch::Loggable required the following methods:
-#
-# * debug?
-# * info?
-# * warn?
-# * error?
-# * fatal?
-
-module Fluent
-  class Log
-    # Elasticsearch::Loggable does not request trace? method.
-    # def trace?
-    #   @level <= LEVEL_TRACE
-    # end
-
-    def debug?
-      @level <= LEVEL_DEBUG
-    end
-
-    def info?
-      @level <= LEVEL_INFO
-    end
-
-    def warn?
-      @level <= LEVEL_WARN
-    end
-
-    def error?
-      @level <= LEVEL_ERROR
-    end
-
-    def fatal?
-      @level <= LEVEL_FATAL
-    end
-  end
-end

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -1,11 +0,0 @@
-module Fluent
-  module ElasticsearchConstants
-    BODY_DELIMITER = "\n".freeze
-    UPDATE_OP = "update".freeze
-    UPSERT_OP = "upsert".freeze
-    CREATE_OP = "create".freeze
-    INDEX_OP = "index".freeze
-    ID_FIELD = "_id".freeze
-    TIMESTAMP_FIELD = "@timestamp".freeze
-  end
-end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -1,89 +0,0 @@
-require 'fluent/event'
-require_relative 'elasticsearch_constants'
-
-class Fluent::ElasticsearchErrorHandler
-  include Fluent::ElasticsearchConstants
-
-  attr_accessor :bulk_message_count
-  class ElasticsearchVersionMismatch < StandardError; end
-  class ElasticsearchError < StandardError; end
-
-  def initialize(plugin)
-    @plugin = plugin
-  end
-
-  def handle_error(response, tag, chunk, bulk_message_count)
-    items = response['items']
-    if items.nil? || !items.is_a?(Array)
-      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
-    end
-    if bulk_message_count != items.length
-        raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
-    end
-    retry_stream = Fluent::MultiEventStream.new
-    stats = Hash.new(0)
-    meta = {}
-    header = {}
-    chunk.msgpack_each do |time, rawrecord|
-      bulk_message = ''
-      next unless rawrecord.is_a? Hash
-      begin
-        # we need a deep copy for process_message to alter
-        processrecord = Marshal.load(Marshal.dump(rawrecord))
-        next unless @plugin.process_message(tag, meta, header, time, processrecord, bulk_message)
-      rescue => e
-        stats[:bad_chunk_record] += 1
-        next
-      end
-      item = items.shift
-      if item.has_key?(@plugin.write_operation)
-        write_operation = @plugin.write_operation
-      elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
-        write_operation = CREATE_OP
-      elsif UPSERT_OP == @plugin.write_operation && item.has_key?(UPDATE_OP)
-        write_operation = UPDATE_OP
-      else
-        # When we don't have an expected ops field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      if item[write_operation].has_key?('status')
-        status = item[write_operation]['status']
-      else
-        # When we don't have a status field, something changed in the API
-        # expected return values (ES 2.x)
-        stats[:errors_bad_resp] += 1
-        next
-      end
-      case
-      when [200, 201].include?(status)
-        stats[:successes] += 1
-      when CREATE_OP == write_operation && 409 == status
-        stats[:duplicates] += 1
-      when 400 == status
-        stats[:bad_argument] += 1
-        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
-      else
-        type = item[write_operation].fetch('error', {})['type']
-        if type
-          stats[type] += 1
-          retry_stream.add(time, rawrecord)
-        else
-          # When we don't have a type field, something changed in the API
-          # expected return values (ES 2.x)
-          stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
-          next
-        end
-        stats[type] += 1
-      end
-    end
-    @plugin.log.on_debug do
-      msg = ["Indexed (op = #{@plugin.write_operation})"]
-      stats.each_pair { |key, value| msg << "#{value} #{key}" }
-      @plugin.log.debug msg.join(', ')
-    end
-    raise Fluent::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
-  end
-end

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -1,10 +0,0 @@
-require 'elasticsearch'
-
-class Fluent::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
-
-  def hosts
-    @transport.logger.debug "In Fluent::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
-    @transport.hosts
-  end
-
-end

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -1,25 +0,0 @@
-require 'securerandom'
-require 'base64'
-require 'fluent/filter'
-
-module Fluent
-  class ElasticsearchGenidFilter < Filter
-    Fluent::Plugin.register_filter('elasticsearch_genid', self)
-
-    config_param :hash_id_key, :string, :default => '_hash'
-
-    def initialize
-      super
-    end
-
-    def configure(conf)
-      super
-    end
-
-    def filter(tag, time, record)
-      record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
-      record
-    end
-
-  end
-end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,264 +0,0 @@
-require 'helper'
-require 'fluent/plugin/out_elasticsearch'
-require 'fluent/plugin/elasticsearch_error_handler'
-require 'json'
-
-class TestElasticsearchErrorHandler < Test::Unit::TestCase
-
-  class TestPlugin
-    attr_reader :log
-    attr_reader :error_events
-    attr_accessor :write_operation
-    def initialize(log)
-      @log = log
-      @write_operation = 'index'
-      @error_events = []
-    end
-
-    def router
-      self
-    end
-
-    def emit_error_event(tag, time, record, e)
-       @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
-    end
-
-    def process_message(tag, meta, header, time, record, bulk_message)
-      if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
-      end
-      return true
-    end
-  end
-
-  class MockChunk
-    def initialize(records)
-      @records = records
-      @index = 0
-    end
-    def msgpack_each
-      @records.each { |item| yield(item[:time],item[:record]) }
-    end
-  end
-
-  def setup
-    Fluent::Test.setup
-    @log_device = Fluent::Test::DummyLogDevice.new
-    if defined?(ServerEngine::DaemonLogger)
-      dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
-      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
-      @log = Fluent::Log.new(logger)
-    else
-      @log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_INFO)
-    end
-    @plugin = TestPlugin.new(@log)
-    @handler = Fluent::ElasticsearchErrorHandler.new(@plugin)
-  end
-
-  def parse_response(value)
-    JSON.parse(value)
-  end
-
-  def test_dlq_400_responses
-    records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
-    response = parse_response(%({
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "status" : 400,
-            "_type"  : "bar",
-              "reason":"unrecognized error"
-            }
-        }
-      ]
-     }))
-    chunk = MockChunk.new(records)
-    @handler.handle_error(response, 'atag', chunk, records.length)
-    assert_equal(1, @plugin.error_events.size)
-    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
-  end
-
-  def create_mock_response(method)
-    parse_response(%({
-      "took" : 1,
-      "errors" : true,
-      "items" : [
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "1",
-            "status" : 201
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "2",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "3",
-            "status" : 409,
-            "error" : {
-              "type":"version_conflict_engine_exception",
-              "reason":"document already exists"
-            }
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "5",
-            "status" : 500,
-            "error" : {
-              "reason":"unrecognized error - no type field"
-            }
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "6",
-            "status" : 429,
-            "error" : {
-              "type" : "es_rejected_execution_exception",
-              "reason":"unable to fulfill request at this time, try again later"
-            }
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "7",
-            "status" : 400,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "#{method}" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "8",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        }
-      ]
-    }))
-  end
-
-  def test_retry_error_index
-    records = []
-    error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
-    10.times do |i|
-      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
-    end
-    chunk = MockChunk.new(records)
-
-    response = create_mock_response('create')
-    begin
-      failed = false
-      @handler.handle_error(response, 'atag', chunk, response['items'].length)
-    rescue Fluent::ElasticsearchOutput::RetryStreamError=>e
-      failed = true
-      records = [].tap do |records|
-        e.retry_stream.each {|time, record| records << record}
-      end
-      assert_equal 3, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 6, records[1]['_id']
-      assert_equal 8, records[2]['_id']
-      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 2, error_ids.length
-      assert_equal 5, error_ids[0]
-      assert_equal 7, error_ids[1]
-      @plugin.error_events.collect {|h| h[:error]}.each do |e|
-        assert_true e.respond_to?(:backtrace)
-      end
-    end
-    assert_true failed
-
-  end
-
-  def test_retry_error_upsert
-    @plugin.write_operation = 'upsert'
-    records = []
-    error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
-    10.times do |i|
-      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
-    end
-    chunk = MockChunk.new(records)
-
-    response = create_mock_response('update')
-    begin
-      failed = false
-      @handler.handle_error(response, 'atag', chunk, response['items'].length)
-    rescue Fluent::ElasticsearchOutput::RetryStreamError=>e
-      failed = true
-      records = [].tap do |records|
-        e.retry_stream.each {|time, record| records << record}
-      end
-      assert_equal 4, records.length
-      assert_equal 2, records[0]['_id']
-      # upsert is retried in case of conflict error.
-      assert_equal 3, records[1]['_id']
-      assert_equal 6, records[2]['_id']
-      assert_equal 8, records[3]['_id']
-      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 2, error_ids.length
-      assert_equal 5, error_ids[0]
-      assert_equal 7, error_ids[1]
-      @plugin.error_events.collect {|h| h[:error]}.each do |e|
-        assert_true e.respond_to?(:backtrace)
-      end
-    end
-    assert_true failed
-  end
-
-  def test_old_es_1_X_responses
-    records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
-    response = parse_response(%({
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "status" : 429,
-            "_type"  : "bar",
-            "error" : "some unrecognized error"
-          }
-        }
-      ]
-     }))
-    chunk = MockChunk.new(records)
-    @handler.handle_error(response, 'atag', chunk, records.length)
-    assert_equal(1, @plugin.error_events.size)
-    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
-  end
-
-end