fluent-plugin-elasticsearch 1.13.4 → 1.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35b11b6ba12d6bfa22a084db14fd4503d2f16769dc9db7d30d22dafa3dbad035
-  data.tar.gz: a0812a380eb81188fa705924ae51df7bbc773b0d565469f48540807bc323ec33
+  metadata.gz: 995e9b0bb1ae8dc5fbf60b80c6d038c0b1f5b696aa4e27842ab23b1609eeef74
+  data.tar.gz: c1b249d0d02c7381b0a115fccad53e68ea170906101913c36a2b52c294bb3275
 SHA512:
-  metadata.gz: 942a5d5e57f7fa81697cdc13ad0024ab40df80d7a2a45a1108cd9acb1909ad87c5b0e4e220ce4df91d35949a12ae413fb0209c59d6f8ddb908beefa2c257799b
-  data.tar.gz: f4f4d3d3c1e0cb2800a31516052a5ca3aecd5ba0c451bcb3dcbf0e91d7e492bbf8da4caf4f1871d24e898276fb8758c76ae52813ade2ab6884bf061de73cff63
+  metadata.gz: 158848be63d3b28213856a45026b65e769e480285b43400a5600a7f5e7d01aed24f44bbbc9fad60d0530f64dad2717cc90d57c0ac84f75c1f46c3f439ab56a5f
+  data.tar.gz: 223f97c244f7488f364d3a553e44b0d1fddf4c3f9c93545dd696e84e6a2bfd92abc8565e42ce7b5c402e4ca9bca242b6ac66a88337165579b6b818a7d3261ff2

data/.gitignore

@@ -16,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .DS_Store
+vendor/

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.14.0
+- introduce dead letter queue to handle issues unpacking file buffer chunks (#398)
+
 ### 1.13.4
 - backport auth: Fix missing auth tokens after reloading connections (#397)
 

data/README.md

@@ -49,6 +49,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [time_parse_error_tag](#time_parse_error_tag)
   + [reconnect_on_error](#reconnect_on_error)
   + [with_transporter_log](#with_transporter_log)
+  + [dlq_handler](#dlq_handler)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
@@ -474,6 +475,17 @@ We recommend to set this true if you start to debug this plugin.
 with_transporter_log true
 ```
 
+### dlq_handler
+Adds an error handler for processing corrupt messages from message buffers.
+There are [known cases](https://bugzilla.redhat.com/show_bug.cgi?id=1562004) where
+fluentd is stuck processing messages because the file buffer is corrupt.  Fluentd
+is unable to clear faulty buffer chunks.
+
+```
+dlq_handler {'type':'drop'}        #default is to log and drop messages
+dlq_handler {'type':'file', 'dir':'/tmp/fluentd/dlq', 'max_files':5, 'max_file_size':104857600}
+```
+
 ### Client/host certificate options
 
 Need to verify Elasticsearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.13.4'
+  s.version       = '1.14.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/dead_letter_queue_drop_handler.rb

@@ -0,0 +1,10 @@
+
+module Fluent::DeadLetterQueueDropHandler
+  def handle_chunk_error(out_plugin, tag, error, time, record)
+    begin
+      log.error("Dropping record from '#{tag}': error:#{error} time:#{time} record:#{record}")
+    rescue=>e
+      log.error("Error while trying to log and drop message from chunk '#{tag}' #{e.message}")
+    end
+  end
+end

data/lib/fluent/plugin/dead_letter_queue_file_handler.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+
+module Fluent::DeadLetterQueueFileHandler
+
+  def handle_chunk_error(out_plugin, tag, error, time, record)
+      begin
+        @dlq_file.info({processed_at: Time.now.utc, tag: tag, error: "#{error.message}", time: time, record: record}.to_json)
+      rescue=>e
+        log.error("Error while trying to log and drop message from chunk '#{tag}' #{e.message}")
+      end
+  end
+
+end
+

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -69,6 +69,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :reconnect_on_error, :bool, :default => false
   config_param :pipeline, :string, :default => nil
   config_param :with_transporter_log, :bool, :default => false
+  config_param :dlq_handler, :hash, :default => { 'type' =>'drop' }
 
   include Fluent::ElasticsearchIndexTemplate
   include Fluent::ElasticsearchConstants
@@ -129,6 +130,44 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
       log_level = conf['@log_level'] || conf['log_level']
       log.warn "Consider to specify log_level with @log_level." unless log_level
     end
+
+    configure_dlq_handler
+
+  end
+
+  def configure_dlq_handler
+    dlq_type = @dlq_handler && @dlq_handler.is_a?(Hash) ? dlq_type = @dlq_handler['type'] : nil
+    return unless dlq_type
+
+    case dlq_type.downcase
+    when 'drop'
+      log.info('Configuring the DROP dead letter queue handler')
+      require_relative 'dead_letter_queue_drop_handler'
+      extend Fluent::DeadLetterQueueDropHandler
+    when 'file'
+      log.info("Configuring the File dead letter queue handler: ")
+      dir = @dlq_handler ['dir'] || '/var/lib/fluentd/dlq'
+      shift_age = @dlq_handler['max_files'] || 0
+      shift_size = @dlq_handler['max_file_size'] || 1048576
+      log.info("Configuring the File dead letter queue handler: ")
+      log.info("                Directory: #{dir}")
+      log.info("  Max number of DLQ files: #{shift_age}")
+      log.info("            Max file size: #{shift_size}")
+      unless Dir.exists?(dir)
+        Dir.mkdir(dir)
+        log.info("Created DLQ directory: '#{dir}'")
+      end
+      require 'logger'
+      require 'json'
+      file = File.join(dir, 'dlq')
+      @dlq_file = Logger.new(file, shift_age, shift_size)
+      @dlq_file.level = Logger::INFO
+      @dlq_file.formatter = proc { |severity, datetime, progname, msg| "#{msg.dump}\n" }
+      log.info ("Created DLQ file #{file}")
+
+      require_relative 'dead_letter_queue_file_handler'
+      extend Fluent::DeadLetterQueueFileHandler
+    end
   end
 
   def create_meta_config_map
@@ -321,76 +360,83 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     chunk.msgpack_each do |time, record|
       @error.records += 1
       next unless record.is_a? Hash
-
-      if @flatten_hashes
-        record = flatten_record(record)
+      begin
+        process_message(tag, meta, header, time, record, bulk_message)
+      rescue=>e
+        handle_chunk_error(self, tag, e, time, record)
       end
+    end
 
-      if @hash_config
-        record = generate_hash_id_key(record)
-      end
+    send_bulk(bulk_message) unless bulk_message.empty?
+    bulk_message.clear
+  end
 
-      dt = nil
-      if @logstash_format || @include_timestamp
-        if record.has_key?(TIMESTAMP_FIELD)
-          rts = record[TIMESTAMP_FIELD]
-          dt = parse_time(rts, time, tag)
-        elsif record.has_key?(@time_key)
-          rts = record[@time_key]
-          dt = parse_time(rts, time, tag)
-          record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
-        else
-          dt = Time.at(time).to_datetime
-          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
-        end
-      end
+  def process_message(tag, meta, header, time, record, bulk_message)
+    if @flatten_hashes
+      record = flatten_record(record)
+    end
 
-      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
-      if target_index_parent && target_index_parent[target_index_child_key]
-        target_index = target_index_parent.delete(target_index_child_key)
-      elsif @logstash_format
-        dt = dt.new_offset(0) if @utc_index
-        target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
+    if @hash_config
+      record = generate_hash_id_key(record)
+    end
+
+    dt = nil
+    if @logstash_format || @include_timestamp
+      if record.has_key?(TIMESTAMP_FIELD)
+        rts = record[TIMESTAMP_FIELD]
+        dt = parse_time(rts, time, tag)
+      elsif record.has_key?(@time_key)
+        rts = record[@time_key]
+        dt = parse_time(rts, time, tag)
+        record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
       else
-        target_index = @index_name
+        dt = Time.at(time).to_datetime
+        record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
       end
+    end
 
-      # Change target_index to lower-case since Elasticsearch doesn't
-      # allow upper-case characters in index names.
-      target_index = target_index.downcase
-      if @include_tag_key
-        record[@tag_key] = tag
-      end
+    target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+    if target_index_parent && target_index_parent[target_index_child_key]
+      target_index = target_index_parent.delete(target_index_child_key)
+    elsif @logstash_format
+      dt = dt.new_offset(0) if @utc_index
+      target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
+    else
+      target_index = @index_name
+    end
 
-      target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
-      if target_type_parent && target_type_parent[target_type_child_key]
-        target_type = target_type_parent.delete(target_type_child_key)
-      else
-        target_type = @type_name
-      end
+    # Change target_index to lower-case since Elasticsearch doesn't
+    # allow upper-case characters in index names.
+    target_index = target_index.downcase
+    if @include_tag_key
+      record[@tag_key] = tag
+    end
 
-      meta.clear
-      meta["_index".freeze] = target_index
-      meta["_type".freeze] = target_type
+    target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
+    if target_type_parent && target_type_parent[target_type_child_key]
+      target_type = target_type_parent.delete(target_type_child_key)
+    else
+      target_type = @type_name
+    end
 
-      if @pipeline
-        meta["pipeline".freeze] = @pipeline
-      end
+    meta.clear
+    meta["_index".freeze] = target_index
+    meta["_type".freeze] = target_type
 
-      @meta_config_map.each do |record_key, meta_key|
-        meta[meta_key] = record[record_key] if record[record_key]
-      end
+    if @pipeline
+      meta["pipeline".freeze] = @pipeline
+    end
 
-      if @remove_keys
-        @remove_keys.each { |key| record.delete(key) }
-      end
+    @meta_config_map.each do |record_key, meta_key|
+      meta[meta_key] = record[record_key] if record[record_key]
+    end
 
-      append_record_to_messages(@write_operation, meta, header, record, bulk_message)
-      @error.bulk_message_count += 1
+    if @remove_keys
+      @remove_keys.each { |key| record.delete(key) }
     end
 
-    send_bulk(bulk_message) unless bulk_message.empty?
-    bulk_message.clear
+    append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+    @error.bulk_message_count += 1
   end
 
   # returns [parent, child_key] of child described by path array in record's tree

data/test/plugin/test_out_elasticsearch.rb

@@ -37,7 +37,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
-      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+        @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
 
@@ -181,6 +181,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
   end
 
+  def assert_logs_include(logs, msg)
+    matches = logs.grep /#{msg}/
+    assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
+  end
+
   def test_configure
     config = %{
       host     logs.google.com
@@ -203,6 +208,38 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass
     assert_false instance.with_transporter_log
+    assert_not_nil instance.dlq_handler
+    assert_equal 'drop', instance.dlq_handler['type']
+  end
+
+  def test_configure_with_dlq_file_handler
+    require 'tmpdir'
+    dir = Dir.mktmpdir
+    config = %Q{
+      host     logs.google.com
+      port     777
+      scheme   https
+      path     /es/
+      user     john
+      password doe
+      dlq_handler {"type":"file", "dir":"#{dir}"}
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 'logs.google.com', instance.host
+    assert_equal 777, instance.port
+    assert_equal 'https', instance.scheme
+    assert_equal '/es/', instance.path
+    assert_equal 'john', instance.user
+    assert_equal 'doe', instance.password
+    assert_equal :TLSv1, instance.ssl_version
+    assert_nil instance.client_key
+    assert_nil instance.client_cert
+    assert_nil instance.client_key_pass
+    assert_false instance.with_transporter_log
+    assert_not_nil instance.dlq_handler
+    assert_equal 'file', instance.dlq_handler['type']
+    assert_true Dir.exists?(dir)
   end
 
   def test_template_already_present
@@ -592,6 +629,30 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
+  def test_write_message_with_dlq_drop_handler
+    driver.configure("target_index_key bad_value\n")
+    log = driver.instance.router.emit_error_handler.log
+    stub_elastic_ping
+    stub_elastic
+    driver.emit({'bad_value'=>"\255"})
+    driver.run
+    assert_logs_include(log.out.logs, 'Dropping')
+  end
+
+  def test_write_message_with_dlq_file_handler
+    log = driver.instance.router.emit_error_handler.log
+    dir = Dir.mktmpdir
+    driver.configure("dlq_handler {\"type\":\"file\", \"dir\":\"#{dir}\"}\n
+      target_index_key bad_value\n
+    ")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit({'bad_value'=>"\255"})
+    driver.run
+    logs = File.readlines(File.join(dir,'dlq'))
+    assert_logs_include(logs, 'invalid')
+  end
+
   def test_writes_to_default_index
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.13.4
+  version: 1.14.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -144,6 +144,8 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
+- lib/fluent/plugin/dead_letter_queue_drop_handler.rb
+- lib/fluent/plugin/dead_letter_queue_file_handler.rb
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb