fluent-plugin-splunk-hec 1.1.2 → 1.2.4

data/Rakefile

@@ -1,5 +1,10 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rake/testtask'
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
 
 Rake::TestTask.new(:test) do |t|
   t.libs << 'test'
@@ -9,4 +14,4 @@ Rake::TestTask.new(:test) do |t|
   t.warning = false
 end
 
-task :default => :test
+task default: :test

data/VERSION

@@ -1 +1 @@
-1.1.2
+1.2.4

data/fluent-plugin-splunk-hec.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-splunk-hec'
   spec.version       = File.read('VERSION')
@@ -31,15 +33,18 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_runtime_dependency 'fluentd', '~> 1.4'
+  spec.add_runtime_dependency 'fluentd', '>= 1.4'
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
-  spec.add_runtime_dependency 'net-http-persistent', '~> 3.0'
+  spec.add_runtime_dependency 'net-http-persistent', '~> 3.1'
+  spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'
+  spec.add_runtime_dependency 'prometheus-client', '< 0.10.0'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rake', '>= 12.0'
   # required by fluent/test.rb
-  spec.add_development_dependency 'test-unit', '~> 3.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
-  spec.add_development_dependency 'webmock', '~> 3.5.0'
+  spec.add_development_dependency 'rubocop', '~> 0.63.1'
   spec.add_development_dependency 'simplecov', '~> 0.16.1'
+  spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 3.5.0'
 end

data/lib/fluent/plugin/out_splunk.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require 'fluent/output'
+require 'fluent/plugin/output'
+require 'fluent/plugin/formatter'
+require 'prometheus/client'
+require 'benchmark'
+
+module Fluent::Plugin
+  class SplunkOutput < Fluent::BufferedOutput
+    helpers :formatter
+
+    autoload :VERSION, 'fluent/plugin/out_splunk/version'
+    autoload :MatchFormatter, 'fluent/plugin/out_splunk/match_formatter'
+
+    KEY_FIELDS = %w[index host source sourcetype metric_name metric_value].freeze
+    TAG_PLACEHOLDER = '${tag}'
+
+    desc 'The host field for events, by default it uses the hostname of the machine that runnning fluentd. This is exclusive with `host_key`.'
+    config_param :host, :string, default: nil
+
+    desc 'Field name to contain host. This is exclusive with `host`.'
+    config_param :host_key, :string, default: nil
+
+    desc 'The source field for events, when not set, will be decided by HEC. This is exclusive with `source_key`.'
+    config_param :source, :string, default: nil
+
+    desc 'Field name to contain source. This is exclusive with `source`.'
+    config_param :source_key, :string, default: nil
+
+    desc 'The sourcetype field for events, when not set, will be decided by HEC. This is exclusive with `sourcetype_key`.'
+    config_param :sourcetype, :string, default: nil
+
+    desc 'Field name to contain sourcetype. This is exclusive with `sourcetype`.'
+    config_param :sourcetype_key, :string, default: nil
+
+    desc 'Field name to contain Splunk event time. By default will use fluentd\'d time'
+    config_param :time_key, :string, default: nil
+
+    desc 'The Splunk index to index events. When not set, will be decided by HEC. This is exclusive with `index_key`'
+    config_param :index, :string, default: nil
+
+    desc 'Field name to contain Splunk index name. This is exclusive with `index`.'
+    config_param :index_key, :string, default: nil
+
+    desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
+    config_param :keep_keys, :bool, default: false
+
+    desc 'Define index-time fields for event data type, or metric dimensions for metric data type. Null value fields will be removed.'
+    config_section :fields, init: false, multi: false, required: false do
+      # this is blank on purpose
+    end
+
+    config_section :format do
+      config_set_default :usage, '**'
+      config_set_default :@type, 'json'
+      config_set_default :add_newline, false
+    end
+
+    desc <<~DESC
+      Whether to allow non-UTF-8 characters in user logs. If set to true, any
+      non-UTF-8 character would be replaced by the string specified by
+      `non_utf8_replacement_string`. If set to false, any non-UTF-8 character
+      would trigger the plugin to error out.
+    DESC
+    config_param :coerce_to_utf8, :bool, default: true
+
+    desc <<~DESC
+      If `coerce_to_utf8` is set to true, any non-UTF-8 character would be
+      replaced by the string specified here.
+    DESC
+    config_param :non_utf8_replacement_string, :string, default: ' '
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      check_conflict
+      @api = construct_api
+      prepare_key_fields
+      configure_fields(conf)
+      configure_metrics(conf)
+
+      # @formatter_configs is from formatter helper
+      @formatters = @formatter_configs.map do |section|
+        MatchFormatter.new section.usage, formatter_create(usage: section.usage)
+      end
+    end
+
+    def write(chunk)
+      log.trace { "#{self.class}: Received new chunk, size=#{chunk.read.bytesize}" }
+
+      t = Benchmark.realtime do
+        write_to_splunk(chunk)
+      end
+
+      @metrics[:record_counter].increment(metric_labels, chunk.size_of_events)
+      @metrics[:bytes_counter].increment(metric_labels, chunk.bytesize)
+      @metrics[:write_records_histogram].observe(metric_labels, chunk.size_of_events)
+      @metrics[:write_bytes_histogram].observe(metric_labels, chunk.bytesize)
+      @metrics[:write_latency_histogram].observe(metric_labels, t)
+    end
+
+    def write_to_splunk(_chunk)
+      raise NotImplementedError("Child class should implement 'write_to_splunk'")
+    end
+
+    def construct_api
+      raise NotImplementedError("Child class should implement 'construct_api'")
+    end
+
+    protected
+
+    def prepare_event_payload(tag, time, record)
+      {
+        host: @host ? @host.call(tag, record) : @default_host,
+        # From the API reference
+        # http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+        # `time` should be a string or unsigned integer.
+        # That's why we use `to_s` here.
+        time: time.to_f.to_s
+      }.tap do |payload|
+        payload[:index] = @index.call(tag, record) if @index
+        payload[:source] = @source.call(tag, record) if @source
+        payload[:sourcetype] = @sourcetype.call(tag, record) if @sourcetype
+
+        # delete nil fields otherwise will get format error from HEC
+        %i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
+
+        if @extra_fields
+          payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
+          payload[:fields].compact!
+          # if a field is already in indexed fields, then remove it from the original event
+          @extra_fields.values.each { |field| record.delete field }
+        end
+        if formatter = @formatters.find { |f| f.match? tag }
+          record = formatter.format(tag, time, record)
+        end
+        payload[:event] = convert_to_utf8 record
+      end
+    end
+
+    def format_event(tag, time, record)
+      MultiJson.dump(prepare_event_payload(tag, time, record))
+    end
+
+    def process_response(response, _request_body)
+      log.trace { "[Response] POST #{@api}: #{response.inspect}" }
+
+      @metrics[:status_counter].increment(metric_labels(status: response.code.to_s))
+
+      # raise Exception to utilize Fluentd output plugin retry mechanism
+      raise "Server error (#{response.code}) for POST #{@api}, response: #{response.body}" if response.code.to_s.start_with?('5')
+
+      # For both success response (2xx) and client errors (4xx), we will consume the chunk.
+      # Because there probably a bug in the code if we get 4xx errors, retry won't do any good.
+      unless response.code.to_s.start_with?('2')
+        log.error "#{self.class}: Failed POST to #{@api}, response: #{response.body}"
+        log.error { "#{self.class}: Failed request body: #{post.body}" }
+      end
+    end
+
+    private
+
+    def check_conflict
+      KEY_FIELDS.each do |f|
+        kf = "#{f}_key"
+        raise Fluent::ConfigError, "Can not set #{f} and #{kf} at the same time." \
+          if %W[@#{f} @#{kf}].all? &method(:instance_variable_get)
+      end
+    end
+
+    def prepare_key_fields
+      KEY_FIELDS.each do |f|
+        v = instance_variable_get "@#{f}_key"
+        if v
+          attrs = v.split('.').freeze
+          if @keep_keys
+            instance_variable_set "@#{f}", ->(_, record) { attrs.inject(record) { |o, k| o[k] } }
+          else
+            instance_variable_set "@#{f}", lambda { |_, record|
+              attrs[0...-1].inject(record) { |o, k| o[k] }.delete(attrs[-1])
+            }
+          end
+        else
+          v = instance_variable_get "@#{f}"
+          next unless v
+
+          if v == TAG_PLACEHOLDER
+            instance_variable_set "@#{f}", ->(tag, _) { tag }
+          else
+            instance_variable_set "@#{f}", ->(_, _) { v }
+          end
+        end
+      end
+    end
+
+    # <fields> directive, which defines:
+    # * when data_type is event, index-time fields
+    # * when data_type is metric, metric dimensions
+    def configure_fields(conf)
+      # This loop looks dump, but it is used to suppress the unused parameter configuration warning
+      # Learned from `filter_record_transformer`.
+      conf.elements.select { |element| element.name == 'fields' }.each do |element|
+        element.each_pair { |k, _v| element.key?(k) }
+      end
+
+      return unless @fields
+
+      @extra_fields = @fields.corresponding_config_element.map do |k, v|
+        [k, v.empty? ? k : v]
+      end.to_h
+      end
+
+    def pick_custom_format_method
+      if @data_type == :event
+        define_singleton_method :format, method(:format_event)
+      else
+        define_singleton_method :format, method(:format_metric)
+      end
+    end
+
+    def configure_metrics(conf)
+      @metric_labels = {
+        type: conf['@type'],
+        plugin_id: plugin_id
+      }
+
+      @metrics = {
+        record_counter: register_metric(::Prometheus::Client::Counter.new(
+                                          :splunk_output_write_records_count,
+                                          'The number of log records being sent'
+                                        )),
+        bytes_counter: register_metric(::Prometheus::Client::Counter.new(
+                                         :splunk_output_write_bytes_count,
+                                         'The number of log bytes being sent'
+                                       )),
+        status_counter: register_metric(::Prometheus::Client::Counter.new(
+                                          :splunk_output_write_status_count,
+                                          'The count of sends by response_code'
+                                        )),
+        write_bytes_histogram: register_metric(::Prometheus::Client::Histogram.new(
+                                                 :splunk_output_write_payload_bytes,
+                                                 'The size of the write payload in bytes', {}, [1024, 23_937, 47_875, 95_750, 191_500, 383_000, 766_000, 1_149_000]
+                                               )),
+        write_records_histogram: register_metric(::Prometheus::Client::Histogram.new(
+                                                   :splunk_output_write_payload_records,
+                                                   'The number of records written per write', {}, [1, 10, 25, 100, 200, 300, 500, 750, 1000, 1500]
+                                                 )),
+        write_latency_histogram: register_metric(::Prometheus::Client::Histogram.new(
+                                                   :splunk_output_write_latency_seconds,
+                                                   'The latency of writes'
+                                                 ))
+      }
+    end
+
+    # Tag metrics with the type string that was used to register the plugin
+    def metric_labels(other_labels = {})
+      @metric_labels.merge other_labels
+    end
+
+    # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
+    # non-UTF-8 character would be replaced by the string specified by
+    # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
+    # non-UTF-8 character would trigger the plugin to error out.
+    # Thanks to
+    # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
+    def convert_to_utf8(input)
+      if input.is_a?(Hash)
+        record = {}
+        input.each do |key, value|
+          record[convert_to_utf8(key)] = convert_to_utf8(value)
+        end
+
+        return record
+      end
+      return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
+      return input unless input.respond_to?(:encode)
+
+      if @coerce_to_utf8
+        input.encode(
+          'utf-8',
+          invalid: :replace,
+          undef: :replace,
+          replace: @non_utf8_replacement_string
+        )
+      else
+        begin
+          input.encode('utf-8')
+        rescue EncodingError
+          log.error do
+            'Encountered encoding issues potentially due to non ' \
+              		     'UTF-8 characters. To allow non-UTF-8 characters and ' \
+              		     'replace them with spaces, please set "coerce_to_utf8" ' \
+              		     'to true.'
+          end
+          raise
+        end
+      end
+    end
+
+    def register_metric(metric)
+      if !@registry.exist?(metric.name)
+        @registry.register(metric)
+      else
+        @registry.get(metric.name)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/{out_splunk_hec → out_splunk}/match_formatter.rb

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 require 'fluent/match'
 
-class Fluent::Plugin::SplunkHecOutput::MatchFormatter
+class Fluent::Plugin::SplunkOutput::MatchFormatter
   def initialize(pattern, formatter)
     # based on fluentd/lib/fluent/event_router.rb
-    patterns = pattern.split(/\s+/).map { |str|
+    patterns = pattern.split(/\s+/).map do |str|
       Fluent::MatchPattern.create(str)
-    }
+    end
     @pattern =
       if patterns.length == 1
         patterns[0]

data/lib/fluent/plugin/out_splunk/version.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Fluent::Plugin::SplunkOutput::VERSION = File.read(File.expand_path('../../../../VERSION', File.dirname(__FILE__))).chomp.strip

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
-
-require "fluent/plugin/output"
-require "fluent/plugin/formatter"
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'fluent/env'
+require 'fluent/output'
+require 'fluent/plugin/output'
+require 'fluent/plugin/formatter'
+require 'fluent/plugin/out_splunk'
 
 require 'openssl'
 require 'multi_json'
 require 'net/http/persistent'
 
 module Fluent::Plugin
-  class SplunkHecOutput < Fluent::Plugin::Output
+  class SplunkHecOutput < SplunkOutput
     Fluent::Plugin.register_output('splunk_hec', self)
 
     helpers :formatter
@@ -19,10 +22,10 @@ module Fluent::Plugin
     KEY_FIELDS = %w[index time host source sourcetype metric_name metric_value].freeze
     TAG_PLACEHOLDER = '${tag}'.freeze
 
-    MISSING_FIELD = Hash.new { |h, k|
+    MISSING_FIELD = Hash.new do |_h, k|
       $log.warn "expected field #{k} but it's missing" if defined?($log)
       MISSING_FIELD
-    }.freeze
+    end.freeze
 
     desc 'Protocol to use to call HEC API.'
     config_param :protocol, :enum, list: %i[http https], default: :https
@@ -69,42 +72,27 @@ module Fluent::Plugin
     desc 'The Splunk index to index events. When not set, will be decided by HEC. This is exclusive with `index_key`'
     config_param :index, :string, default: nil
 
-    desc 'Field name to contain Splunk event time. By default will use fluentd\'d time'
-    config_param :time_key, :string, default: nil
-
     desc 'Field name to contain Splunk index name. This is exclusive with `index`.'
     config_param :index_key, :string, default: nil
 
-    desc "The host field for events, by default it uses the hostname of the machine that runnning fluentd. This is exclusive with `host_key`."
-    config_param :host, :string, default: nil
-
-    desc 'Field name to contain host. This is exclusive with `host`.'
-    config_param :host_key, :string, default: nil
-
-    desc 'The source field for events, when not set, will be decided by HEC. This is exclusive with `source_key`.'
-    config_param :source, :string, default: nil
-
-    desc 'Field name to contain source. This is exclusive with `source`.'
-    config_param :source_key, :string, default: nil
-
-    desc 'The sourcetype field for events, when not set, will be decided by HEC. This is exclusive with `sourcetype_key`.'
-    config_param :sourcetype, :string, default: nil
-
-    desc 'Field name to contain sourcetype. This is exclusive with `sourcetype`.'
-    config_param :sourcetype_key, :string, default: nil
-
     desc 'When `data_type` is set to "metric", by default it will treat every key-value pair in the income event as a metric name-metric value pair. Set `metrics_from_event` to `false` to disable this behavior and use `metric_name_key` and `metric_value_key` to define metrics.'
     config_param :metrics_from_event, :bool, default: true
 
-    desc "Field name to contain metric name. This is exclusive with `metrics_from_event`, when this is set, `metrics_from_event` will be set to `false`."
+    desc 'Field name to contain metric name. This is exclusive with `metrics_from_event`, when this is set, `metrics_from_event` will be set to `false`.'
     config_param :metric_name_key, :string, default: nil
 
-    desc "Field name to contain metric value, this is required when `metric_name_key` is set."
+    desc 'Field name to contain metric value, this is required when `metric_name_key` is set.'
     config_param :metric_value_key, :string, default: nil
 
     desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
     config_param :keep_keys, :bool, default: false
 
+    desc 'App name'
+    config_param :app_name, :string, default: "hec_plugin_gem"
+
+    desc 'App version'
+    config_param :app_version, :string, default: "#{VERSION}"
+
     desc 'Define index-time fields for event data type, or metric dimensions for metric data type. Null value fields will be removed.'
     config_section :fields, init: false, multi: false, required: false do
       # this is blank on purpose
@@ -139,47 +127,45 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      check_conflict
       check_metric_configs
-      construct_api
-      prepare_key_fields
-      configure_fields(conf)
       pick_custom_format_method
-
-      # @formatter_configs is from formatter helper
-      @formatters = @formatter_configs.map { |section|
-	MatchFormatter.new section.usage, formatter_create(usage: section.usage)
-      }
     end
 
     def start
       super
+      @conn = Net::HTTP::Persistent.new.tap do |c|
+        c.verify_mode = @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+        c.cert = OpenSSL::X509::Certificate.new File.read(@client_cert) if @client_cert
+        c.key = OpenSSL::PKey::RSA.new File.read(@client_key) if @client_key
+        c.ca_file = @ca_file
+        c.ca_path = @ca_path
+        c.ciphers = @ssl_ciphers
+
+        c.override_headers['Content-Type'] = 'application/json'
+        c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}"
+        c.override_headers['Authorization'] = "Splunk #{@hec_token}"
+        c.override_headers['__splunk_app_name'] = "#{@app_name}"
+        c.override_headers['__splunk_app_version'] = "#{@app_version}"
 
-      @hec_conn = new_connection
+      end
     end
 
-    def format(tag, time, record)
-      # this method will be replaced in `configure`
+    def shutdown
+      super
+      @conn.shutdown
     end
 
-    def write(chunk)
-      log.debug { "Received new chunk, size=#{chunk.read.bytesize}" }
-      send_to_hec chunk
+    def format(tag, time, record)
+      # this method will be replaced in `configure`
     end
 
     def multi_workers_ready?
       true
     end
 
-    private
+    protected
 
-    def check_conflict
-      KEY_FIELDS.each { |f|
-	kf = "#{f}_key"
-	raise Fluent::ConfigError, "Can not set #{f} and #{kf} at the same time." \
-	  if %W[@#{f} @#{kf}].all? &method(:instance_variable_get)
-      }
-    end
+    private
 
     def check_metric_configs
       return unless @data_type == :metric
@@ -188,71 +174,57 @@ module Fluent::Plugin
 
       return if @metrics_from_event
 
-      raise Fluent::ConfigError, "`metric_name_key` is required when `metrics_from_event` is `false`." unless @metric_name_key
-
-      raise Fluent::ConfigError, "`metric_value_key` is required when `metric_name_key` is set." unless @metric_value_key
-    end
-
-    def prepare_key_fields
-      KEY_FIELDS.each { |f|
-	v = instance_variable_get "@#{f}_key"
-	if v
-	  attrs = v.split('.').freeze
-	  if @keep_keys
-	    instance_variable_set "@#{f}", ->(_, record) { attrs.inject(record) { |o, k| o[k] } }
-	  else
-	    instance_variable_set "@#{f}", ->(_, record) {
-	      attrs[0...-1].inject(record) { |o, k| o[k] }.delete(attrs[-1])
-	    }
-	  end
-	else
-	  v = instance_variable_get "@#{f}"
-	  next unless v
-
-	  if v == TAG_PLACEHOLDER
-	    instance_variable_set "@#{f}", ->(tag, _) { tag }
-	  else
-	    instance_variable_set "@#{f}", ->(_, _) { v }
-	  end
-	end
-      }
+      raise Fluent::ConfigError, '`metric_name_key` is required when `metrics_from_event` is `false`.' unless @metric_name_key
+      raise Fluent::ConfigError, '`metric_value_key` is required when `metric_name_key` is set.' unless @metric_value_key
     end
 
-    # <fields> directive, which defines:
-    # * when data_type is event, index-time fields
-    # * when data_type is metric, metric dimensions
-    def configure_fields(conf)
-      # This loop looks dump, but it is used to suppress the unused parameter configuration warning
-      # Learned from `filter_record_transformer`.
-      conf.elements.select { |element| element.name == 'fields' }.each do |element|
-        element.each_pair { |k, v| element.has_key?(k) }
-      end
+    def format_event(tag, time, record)
+      MultiJson.dump({
+        host: @host ? @host.(tag, record) : @default_host,
+        # From the API reference
+        # http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+        # `time` should be a string or unsigned integer.
+        # That's why we use the to_string function here.
+        time: time.to_f.to_s
+        }.tap { |payload|
+          if @time
+            time_value = @time.(tag, record)
+            # if no value is found don't override and use fluentd's time
+            if !time_value.nil?
+              payload[:time] = time_value
+            end
+          end
 
-      return unless @fields
+          payload[:index] = @index.(tag, record) if @index
+          payload[:source] = @source.(tag, record) if @source
+          payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
 
-      @extra_fields = @fields.corresponding_config_element.map { |k, v|
-	[k, v.empty? ? k : v]
-      }.to_h
-    end
+          # delete nil fields otherwise will get formet error from HEC
+          %i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
 
-    def pick_custom_format_method
-      if @data_type == :event
-	define_singleton_method :format, method(:format_event)
-      else
-	define_singleton_method :format, method(:format_metric)
-      end
+          if @extra_fields
+            payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
+            payload[:fields].delete_if { |_k,v| v.nil? }
+            # if a field is already in indexed fields, then remove it from the original event
+            @extra_fields.values.each { |field| record.delete field }
+          end
+          if formatter = @formatters.find { |f| f.match? tag }
+            record = formatter.format(tag, time, record)
+          end
+          payload[:event] = convert_to_utf8 record
+      })
     end
 
-    def format_event(tag, time, record)
-      MultiJson.dump({
-	host: @host ? @host.(tag, record) : @default_host,
-	# From the API reference
-	# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
-	# `time` should be a string or unsigned integer.
-	# That's why we use the to_string function here.
-	time: time.to_f.to_s
-      }.tap { |payload|
-
+    def format_metric(tag, time, record)
+      payload = {
+        host: @host ? @host.call(tag, record) : @default_host,
+        # From the API reference
+        # http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+        # `time` should be a string or unsigned integer.
+        # That's why we use `to_s` here.
+        time: time.to_f.to_s,
+        event: 'metric'
+      }.tap do |payload| 
         if @time
           time_value = @time.(tag, record)
           # if no value is found don't override and use fluentd's time
@@ -260,71 +232,42 @@ module Fluent::Plugin
             payload[:time] = time_value
           end
         end
+      end
+      payload[:index] = @index.call(tag, record) if @index
+      payload[:source] = @source.call(tag, record) if @source
+      payload[:sourcetype] = @sourcetype.call(tag, record) if @sourcetype
+
+      unless @metrics_from_event
+        fields = {
+          metric_name: @metric_name.call(tag, record),
+          _value: @metric_value.call(tag, record)
+        }
+
+        if @extra_fields
+          fields.update @extra_fields.map { |name, field| [name, record[field]] }.to_h
+          fields.delete_if { |_k,v| v.nil? }
+        else
+          fields.update record
+        end
 
-	payload[:index] = @index.(tag, record) if @index
-	payload[:source] = @source.(tag, record) if @source
-	payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
-
-	# delete nil fields otherwise will get formet error from HEC
-	%i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
-
-	if @extra_fields
-    payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
-    payload[:fields].delete_if { |_k,v| v.nil? }
-	  # if a field is already in indexed fields, then remove it from the original event
-	  @extra_fields.values.each { |field| record.delete field }
-	end
-	if formatter = @formatters.find { |f| f.match? tag }
-	  record = formatter.format(tag, time, record)
-	end
-	payload[:event] = convert_to_utf8 record
-      })
-    end
+        fields.delete_if { |_k,v| v.nil? }
 
-    def format_metric(tag, time, record)
-      payload = {
-	host: @host ? @host.(tag, record) : @default_host,
-	# From the API reference
-	# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
-	# `time` should be a string or unsigned integer.
-	# That's why we use `to_s` here.
-	time: time.to_f.to_s,
-	event: 'metric'
-      }
-      payload[:index] = @index.(tag, record) if @index
-      payload[:source] = @source.(tag, record) if @source
-      payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
-
-      if not @metrics_from_event
-	fields = {
-	  metric_name: @metric_name.(tag, record),
-	  _value: @metric_value.(tag, record)
-	}
-
-	if @extra_fields
-	  fields.update @extra_fields.map { |name, field| [name, record[field]] }.to_h
-	else
-	  fields.update record
-	end
-
-  fields.delete_if { |_k,v| v.nil? }
-
-	payload[:fields] = convert_to_utf8 fields
-
-	return MultiJson.dump(payload)
+        payload[:fields] = convert_to_utf8 fields
+
+        return MultiJson.dump(payload)
       end
 
       # when metrics_from_event is true, generate one metric event for each key-value in record
-      payloads = record.map { |key, value|
-	{fields: {metric_name: key, _value: value}}.merge! payload
-      }
+      payloads = record.map do |key, value|
+        { fields: { metric_name: key, _value: value } }.merge! payload
+      end
 
       payloads.map!(&MultiJson.method(:dump)).join
     end
 
     def construct_api
-      @hec_api = URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
-    rescue
+      URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
+    rescue StandardError
       raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid."
     end
 
@@ -343,27 +286,34 @@ module Fluent::Plugin
         c.override_headers['Content-Type'] = 'application/json'
         c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}"
         c.override_headers['Authorization'] = "Splunk #{@hec_token}"
+        c.override_headers['__splunk_app_name'] = "#{@app_name}"
+        c.override_headers['__splunk_app_version'] = "#{@app_version}"
+
       end
     end
 
-    def send_to_hec(chunk)
-      post = Net::HTTP::Post.new @hec_api.request_uri
+    def write_to_splunk(chunk)
+      post = Net::HTTP::Post.new @api.request_uri
       post.body = chunk.read
-      log.debug { "Sending #{post.body.bytesize} bytes to Splunk." }
+      log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
+      log.trace { "POST #{@api} body=#{post.body}" }
 
-      log.trace { "POST #{@hec_api} body=#{post.body}" }
-      response = @hec_conn.request @hec_api, post
-      log.debug { "[Response] POST #{@hec_api}: #{response.inspect}" }
+      t1 = Time.now
+      response = @conn.request @api, post
+      t2 = Time.now
 
       # raise Exception to utilize Fluentd output plugin retry machanism
-      raise "Server error (#{response.code}) for POST #{@hec_api}, response: #{response.body}" if response.code.start_with?('5')
+      raise "Server error (#{response.code}) for POST #{@api}, response: #{response.body}" if response.code.start_with?('5')
 
       # For both success response (2xx) and client errors (4xx), we will consume the chunk.
       # Because there probably a bug in the code if we get 4xx errors, retry won't do any good.
       if not response.code.start_with?('2')
-	log.error "Failed POST to #{@hec_api}, response: #{response.body}"
-	log.debug { "Failed request body: #{post.body}" }
+        log.error "Failed POST to #{@api}, response: #{response.body}"
+        log.debug { "Failed request body: #{post.body}" }
       end
+
+      log.debug { "[Response] Chunk: #{dump_unique_id_hex(chunk.unique_id)} Size: #{post.body.bytesize} Response: #{response.inspect} Duration: #{t2 - t1}" }
+      process_response(response, post.body)
     end
 
     # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
@@ -374,32 +324,32 @@ module Fluent::Plugin
     # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
     def convert_to_utf8(input)
       if input.is_a?(Hash)
-	record = {}
-	input.each do |key, value|
-	  record[convert_to_utf8(key)] = convert_to_utf8(value)
-	end
+        record = {}
+        input.each do |key, value|
+          record[convert_to_utf8(key)] = convert_to_utf8(value)
+        end
 
-	return record
+        return record
       end
       return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
       return input unless input.respond_to?(:encode)
 
       if @coerce_to_utf8
-	input.encode(
-	  'utf-8',
-	  invalid: :replace,
-	  undef: :replace,
-	  replace: @non_utf8_replacement_string)
-      else
-	begin
-	  input.encode('utf-8')
-	rescue EncodingError
-	  log.error { 'Encountered encoding issues potentially due to non ' \
-		     'UTF-8 characters. To allow non-UTF-8 characters and ' \
-		     'replace them with spaces, please set "coerce_to_utf8" ' \
-		     'to true.' }
-	  raise
-  end
+        input.encode(
+          'utf-8',
+          invalid: :replace,
+          undef: :replace,
+          replace: @non_utf8_replacement_string)
+            else
+        begin
+          input.encode('utf-8')
+        rescue EncodingError
+          log.error { 'Encountered encoding issues potentially due to non ' \
+              'UTF-8 characters. To allow non-UTF-8 characters and ' \
+              'replace them with spaces, please set "coerce_to_utf8" ' \
+              'to true.' }
+          raise
+        end
       end
     end
   end