fluent-plugin-splunk-http-eventcollector-test 0.3.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7ca947bc9db8558af7c0078b36ce1dddced57881
+  data.tar.gz: 0e860bbd428a4b4884b5102b5f273b0b67417ffe
+SHA512:
+  metadata.gz: e26e4c0c9517980df6de47fa920bf704f0222c81e44fe37abb67baa490b8d5433414ef5dbc1f4081ade5f2fb8fa244f7035fa74c44bd1b6bc34689de9e9aff1b
+  data.tar.gz: 73e65d9446f5a197ad25a9350299a3328fd387b2a4d0c7580a2d593ceed6f308e4c6b01abb6ed5e7b5bbe2aa847d8649d98b6453ea790a9b9289ef8f719a47f7

data/.gitignore

@@ -0,0 +1,27 @@
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+Gemfile.lock
+.ruby-version
+.ruby-gemset

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-splunk-http-eventcollector.gemspec
+gemspec
+
+#gem "test-unit"

data/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2015, Bryce Chidester (Calyptix Security)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/README.md

@@ -0,0 +1,206 @@
+# Fluent::Plugin::SplunkHTTPEventcollector, a plugin for [Fluentd](http://fluentd.org)
+
+Splunk output plugin for Fluent event collector.
+
+This plugin interfaces with the Splunk HTTP Event Collector:
+  http://dev.splunk.com/view/event-collector/SP-CAAAE6M
+
+[![Build Status](https://travis-ci.org/brycied00d/fluent-plugin-splunk-http-eventcollector.svg?branch=master)](https://travis-ci.org/brycied00d/fluent-plugin-splunk-http-eventcollector)
+
+## Basic Example
+
+    <match **>
+      type splunk-http-eventcollector
+      server 127.0.0.1:8088
+      verify false
+      token YOUR-TOKEN
+
+      # Convert fluent tags to Splunk sources.
+      # If you set an index, "check_index false" is required.
+      host YOUR-HOSTNAME
+      index SOME-INDEX
+      check_index false
+      source {TAG}
+      sourcetype fluent
+
+      # TIMESTAMP: key1="value1" key2="value2" ...
+      time_format unixtime
+      format kvp
+
+      # Memory buffer with a short flush internal.
+      buffer_type memory
+      buffer_queue_limit 16
+      buffer_chunk_limit 8m
+      flush_interval 2s
+    </match>
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-splunk-http-eventcollector'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-splunk-http-eventcollector
+
+Whatever is appropriate for your environment. Note: If you're using the
+`td-agent` package, it brings with it its own "embedded" Ruby environment with
+either `td-agent-gem` or `/opt/td-agent/embedded/bin/gem` depending on platform.
+
+## Configuration
+
+Put the following lines to your fluent.conf:
+
+    <match **>
+      type splunk-http-eventcollector
+
+      # server: Splunk server host and port
+      # default: localhost:8088
+      server localhost:8088
+
+      # protocol: Connect to Splunk server via 'http' or 'https'
+      # default: https
+      #protocol http
+
+      # verify: SSL server verification
+      # default: true
+      #verify false
+
+      # token: the token issued
+      token YOUR-TOKEN
+
+      #
+      # Event Parameters
+      #
+
+      # host: 'host' parameter passed to Splunk
+      host YOUR-HOSTNAME
+
+      # index: 'index' parameter passed to Splunk (REST only)
+      # default: <none>
+      #index main
+
+      # check_index: 'check-index' parameter passed to Splunk (REST only)
+      # default: <none>
+      #check_index false
+
+      # host: 'source' parameter passed to Splunk
+      # default: {TAG}
+      #
+      # "{TAG}" will be replaced by fluent tags at runtime
+      source {TAG}
+
+      # sourcetype: 'sourcetype' parameter passed to Splunk
+      # default: fluent
+      sourcetype fluent
+
+      #
+      # Formatting Parameters
+      #
+
+      # time_format: the time format of each event
+      # value: none, unixtime, localtime, or any time format string
+      # default: localtime
+      time_format localtime
+
+      # format: the text format of each event
+      # value: json, kvp, or text
+      # default: json
+      #
+      # input = {"x":1, "y":"xyz", "message":"Hello, world!"}
+      # 
+      # 'json' is JSON encoding:
+      #   {"x":1,"y":"xyz","message":"Hello, world!"}
+      # 
+      # 'kvp' is "key=value" pairs, which is automatically detected as fields by Splunk:
+      #   x="1" y="xyz" message="Hello, world!"
+      # 
+      # 'text' outputs the value of "message" as is, with "key=value" pairs for others:
+      #   [x="1" y="xyz"] Hello, world!
+      format json
+
+      #
+      # Buffering Parameters
+      #
+
+      # Standard parameters for buffering.  See documentation for details:
+      #   http://docs.fluentd.org/articles/buffer-plugin-overview
+      buffer_type memory
+      buffer_queue_limit 16
+
+      # buffer_chunk_limit: The maxium size of POST data in a single API call.
+      # 
+      # This value should be reasonablly small since the current implementation
+      # of out_splunk-http-eventcollector converts a chunk to POST data on memory before API calls.
+      # The default value should be good enough.
+      buffer_chunk_limit 8m
+
+      # flush_interval: The interval of API requests.
+      # 
+      # Make sure that this value is sufficiently large to make successive API calls.
+      # Note that a different 'source' creates a different API POST, each of which may
+      # take two or more seconds.  If you include "{TAG}" in the source parameter and
+      # this 'match' section recieves many tags, a single flush may take long time.
+      # (Run fluentd with -v to see verbose logs.)
+      flush_interval 60s
+    </match>
+
+## Example
+
+    # Input from applications
+    <source>
+      type forward
+    </source>
+
+    # Input from log files
+    <source>
+      type tail
+      path /var/log/apache2/ssl_access.log
+      tag ssl_access.log
+      format /(?<message>.*)/
+      pos_file /var/log/td-agent/ssl_access.log.pos
+    </source>
+
+    # fluent logs in text format
+    <match fluent.*>
+      type splunk-http-eventcollector
+      protocol rest
+      server splunk.example.com:8089
+      auth admin:pass
+      sourcetype fluentd
+      format text
+    </match>
+
+    # log files in text format without timestamp
+    <match *.log>
+      type splunk-http-eventcollector
+      protocol rest
+      server splunk.example.com:8089
+      auth admin:pass
+      sourcetype log
+      time_format none
+      format text
+    </match>
+
+    # application logs in kvp format
+    <match app.**>
+      type splunk-http-eventcollector
+      protocol rest
+      server splunk.example.com:8089
+      auth admin:pass
+      sourcetype app
+      format kvp
+    </match>
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-splunk-http-eventcollector-test.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name             = "fluent-plugin-splunk-http-eventcollector-test"
+  gem.version          = "0.3.0.1"
+  gem.authors          = ["Bryce Chidester"]
+  gem.email            = ["bryce.chidester@calyptix.com"]
+  gem.summary          = "Splunk output plugin for Fluentd with increased batch_ size_limit"
+  gem.description      = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
+  gem.homepage         = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
+  gem.license          = 'BSD-2-Clause'
+  gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
+  gem.files            = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
+                           "Rakefile", "test/helper.rb",
+                           "fluent-plugin-splunk-http-eventcollector-test.gemspec",
+                           "lib/fluent/plugin/out_splunk-http-eventcollector.rb",
+                           "test/plugin/test_out_splunk-http-eventcollector.rb" ]
+  gem.test_files       = [ "test/helper.rb",
+                           "test/plugin/test_out_splunk-http-eventcollector.rb" ]
+  gem.require_paths    = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", '~> 3.1'
+  gem.add_development_dependency "webmock", '~> 2.3', '>= 2.3.2'
+  gem.add_runtime_dependency "fluentd", '~> 0.12.12'
+  gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
+end

data/lib/fluent/plugin/out_splunk-http-eventcollector.rb

@@ -0,0 +1,325 @@
+=begin
+
+Copyright (c) 2015, Bryce Chidester (Calyptix Security)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=end
+
+# Splunk HTTP Event collector docs
+# http://dev.splunk.com/view/event-collector/SP-CAAAE6M
+# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+
+module Fluent
+class SplunkHTTPEventcollectorOutput < BufferedOutput
+
+  Plugin.register_output('splunk-http-eventcollector', self)
+
+  config_param :test_mode, :bool, :default => false
+
+  config_param :server, :string, :default => 'localhost:8088'
+  config_param :verify, :bool, :default => true
+  config_param :token, :string, :default => nil
+
+  # Event parameters
+  config_param :protocol, :string, :default => 'https'
+  config_param :host, :string, :default => nil
+  config_param :index, :string, :default => 'main'
+  config_param :all_items, :bool, :default => false
+
+  config_param :sourcetype, :string, :default => 'fluentd'
+  config_param :source, :string, :default => nil
+  config_param :post_retry_max, :integer, :default => 5
+  config_param :post_retry_interval, :integer, :default => 5
+
+  # TODO Find better upper limits
+  #config_param :batch_size_limit, :integer, :default => 262144 # 65535
+  config_param :batch_size_limit, :integer, :default => 10485760
+  #config_param :batch_event_limit, :integer, :default => 100
+
+  # Whether to allow non-UTF-8 characters in user logs. If set to true, any
+  # non-UTF-8 character would be replaced by the string specified by
+  # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
+  # would trigger the plugin to error out.
+  config_param :coerce_to_utf8, :bool, :default => true
+
+  # If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be
+  # replaced by the string specified here.
+  config_param :non_utf8_replacement_string, :string, :default => ' '
+
+  # Called on class load (class initializer)
+  def initialize
+    super
+    log.trace "splunk-http-eventcollector(initialize) called"
+    require 'net/http/persistent'
+    require 'openssl'
+  end  # initialize
+
+  # Thanks to
+  # https://github.com/kazegusuri/fluent-plugin-prometheus/blob/348c112d/lib/fluent/plugin/prometheus.rb
+  def self.placeholder_expander(log)
+    # Use internal class in order to expand placeholder
+    if defined?(Fluent::Filter) # for v0.12, built-in PlaceholderExpander
+      begin
+        require 'fluent/plugin/filter_record_transformer'
+        if defined?(Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander)
+          # for v0.14
+          return Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        else
+          # for v0.12
+          return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        end
+      rescue LoadError => e
+        raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
+      end
+    else # for v0.10, use PlaceholderExapander in fluent-plugin-record-reformer plugin
+      begin
+        require 'fluent/plugin/out_record_reformer.rb'
+        return Fluent::RecordReformerOutput::PlaceholderExpander.new(log: log)
+      rescue LoadError => e
+        raise ConfigError, "cannot find fluent-plugin-record-reformer: #{e.message}"
+      end
+    end
+  end
+
+  ## This method is called before starting.
+  ## 'conf' is a Hash that includes configuration parameters.
+  ## If the configuration is invalid, raise Fluent::ConfigError.
+  def configure(conf)
+    super
+    log.trace "splunk-http-eventcollector(configure) called"
+    begin
+      @splunk_uri = URI "#{@protocol}://#{@server}/services/collector"
+    rescue
+      raise ConfigError, "Unable to parse the server into a URI."
+    end
+
+    @placeholder_expander = Fluent::SplunkHTTPEventcollectorOutput.placeholder_expander(log)
+    @hostname = Socket.gethostname
+    # TODO Add other robust input/syntax checks.
+  end  # configure
+
+  ## This method is called when starting.
+  ## Open sockets or files here.
+  def start
+    super
+    log.trace "splunk-http-eventcollector(start) called"
+    @http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
+    @http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
+    @http.override_headers['Content-Type'] = 'application/json'
+    @http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
+    @http.override_headers['Authorization'] = "Splunk #{@token}"
+
+    log.trace "initialized for splunk-http-eventcollector"
+  end
+
+  ## This method is called when shutting down.
+  ## Shutdown the thread and close sockets or files here.
+  def shutdown
+    super
+    log.trace "splunk-http-eventcollector(shutdown) called"
+
+    @http.shutdown
+    log.trace "shutdown from splunk-http-eventcollector"
+  end  # shutdown
+
+  ## This method is called when an event reaches to Fluentd. (like unbuffered emit())
+  ## Convert the event to a raw string.
+  def format(tag, time, record)
+    #log.trace "splunk-http-eventcollector(format) called"
+    # Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
+
+    placeholder_values = {
+      'tag' => tag,
+      'tag_parts' => tag.split('.'),
+      'hostname' => @hostname,
+      'record' => record
+    }
+
+    placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
+
+    splunk_object = Hash[
+        "time" => time.to_i,
+        "source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source, placeholders) end,
+        "sourcetype" => @placeholder_expander.expand(@sourcetype.to_s, placeholders),
+        "host" => @placeholder_expander.expand(@host.to_s, placeholders),
+        "index" =>  @placeholder_expander.expand(@index, placeholders)
+      ]
+    # TODO: parse different source types as expected: KVP, JSON, TEXT
+    if @all_items
+      splunk_object["event"] = convert_to_utf8(record)
+    else
+      splunk_object["event"] = convert_to_utf8(record["message"])
+    end
+
+    json_event = splunk_object.to_json
+    #log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
+    #log.debug "format: returning: #{[tag, record].to_json.to_s}"
+    json_event
+  end
+
+  # By this point, fluentd has decided its buffer is full and it's time to flush
+  # it. chunk.read is a concatenated string of JSON.to_s objects. Simply POST
+  # them to Splunk and go about our life.
+  ## This method is called every flush interval. Write the buffer chunk
+  ## to files or databases here.
+  ## 'chunk' is a buffer chunk that includes multiple formatted
+  ## events. You can use 'data = chunk.read' to get all events and
+  ## 'chunk.open {|io| ... }' to get IO objects.
+  ##
+  ## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
+  def write(chunk)
+    log.trace "splunk-http-eventcollector(write) called"
+
+    # Break the concatenated string of JSON-formatted events into an Array
+    split_chunk = chunk.read.split("}{").each do |x|
+      # Reconstruct the opening{/closing} that #split() strips off.
+      x.prepend("{") unless x.start_with?("{")
+      x << "}" unless x.end_with?("}")
+    end
+    log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
+        "#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
+    # If fluentd is pushing too much data to Splunk at once, split up the payload
+    # Don't care about the number of events so much as the POST size (bytes)
+    #if split_chunk.size > @batch_event_limit
+    #  log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
+    #      "events in a single push to Splunk. The configured limit is " +
+    #      "#{numfmt(@batch_event_limit)}."
+    #end
+    if chunk.read.bytesize > @batch_size_limit
+      log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
+          "bytes in a single push to Splunk. The configured limit is " +
+          "#{numfmt(@batch_size_limit)} bytes."
+      newbuffer = Array.new
+      split_chunk_counter = 0
+      split_chunk.each do |c|
+        split_chunk_counter = split_chunk_counter + 1
+        #log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
+        #    "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
+        #    "c.bytesize=#{numfmt(c.bytesize)} ????"
+        if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
+          #log.debug "Appended!"
+          newbuffer << c
+        else
+          # Reached the limit - push the current newbuffer.join, and reset
+          #log.debug "Would exceed limit. Flushing newbuffer and continuing."
+          log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
+              "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
+              "c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
+              "flushing current buffer to Splunk."
+          push_buffer newbuffer.join
+          newbuffer = Array c
+        end # if/else buffer fits limit
+      end # split_chunk.each
+      # Push anything left over.
+      push_buffer newbuffer.join if newbuffer.size
+      return
+    else
+      return push_buffer chunk.read
+    end # if chunk.read.bytesize > @batch_size_limit
+  end # write
+
+  def push_buffer(body)
+    post = Net::HTTP::Post.new @splunk_uri.request_uri
+    post.body = body
+    log.debug "POST #{@splunk_uri}"
+    if @test_mode
+      log.debug "TEST_MODE Payload: #{body}"
+      return
+    end
+    # retry up to :post_retry_max times
+    1.upto(@post_retry_max) do |c|
+      response = @http.request @splunk_uri, post
+      log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
+          "(#{response.message})"
+      # TODO check the actual server response too (it's JSON)
+      if response.code == "200"  # and...
+        # success
+        break
+      # TODO check 40X response within post_retry_max and retry
+      elsif response.code.match(/^50/) and c < @post_retry_max
+        # retry
+        log.warn "#{@splunk_uri}: Server error #{response.code} (" +
+            "#{response.message}). Retrying in #{@post_retry_interval} " +
+            "seconds.\n#{response.body}"
+        sleep @post_retry_interval
+        next
+      elsif response.code.match(/^40/)
+        # user error
+        log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
+        break
+      elsif c < @post_retry_max
+        # retry
+        log.debug "#{@splunk_uri}: Retrying..."
+        sleep @post_retry_interval
+        next
+      else
+        # other errors. fluentd will retry processing on exception
+        # FIXME: this may duplicate logs when using multiple buffers
+        raise "#{@splunk_uri}: #{response.message}\n#{response.body}"
+      end # If response.code
+    end # 1.upto(@post_retry_max)
+  end # push_buffer
+
+  def numfmt(input)
+    input.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\1,').reverse
+  end # numfmt
+
+  # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
+  # non-UTF-8 character would be replaced by the string specified by
+  # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
+  # non-UTF-8 character would trigger the plugin to error out.
+  # Thanks to
+  # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
+  def convert_to_utf8(input)
+    if input.is_a?(Hash)
+      record = {}
+      input.each do |key, value|
+        record[convert_to_utf8(key)] = convert_to_utf8(value)
+      end
+
+      return record
+    end
+    return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
+    return input unless input.respond_to?(:encode)
+
+    if @coerce_to_utf8
+      input.encode(
+        'utf-8',
+        invalid: :replace,
+        undef: :replace,
+        replace: @non_utf8_replacement_string)
+    else
+      begin
+        input.encode('utf-8')
+      rescue EncodingError
+        @log.error 'Encountered encoding issues potentially due to non ' \
+                   'UTF-8 characters. To allow non-UTF-8 characters and ' \
+                   'replace them with spaces, please set "coerce_to_utf8" ' \
+                   'to true.'
+        raise
+      end
+    end
+  end
+end  # class SplunkHTTPEventcollectorOutput
+end  # module Fluent

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'webmock/test_unit'
+require 'fluent/plugin/out_splunk-http-eventcollector'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_splunk-http-eventcollector.rb

@@ -0,0 +1,179 @@
+require 'helper'
+
+class SplunkHTTPEventcollectorOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    server localhost:8089
+    verify false
+    token changeme
+  ]
+
+  def create_driver(conf=CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHTTPEventcollectorOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    # default
+    d = create_driver
+    assert_equal nil, d.instance.source
+    assert_equal 'fluentd', d.instance.sourcetype
+  end
+
+  def test_write
+    stub_request(:post, "https://localhost:8089/services/collector").
+      to_return(body: '{"text":"Success","code":0}')
+
+    d = create_driver
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({ "message" => "a message"}, time)
+
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {
+        "Authorization" => "Splunk changeme",
+        'Content-Type' => 'application/json',
+        'User-Agent' => 'fluent-plugin-splunk-http-eventcollector/0.0.1'
+      },
+      body: { time: time, source:"test", sourcetype: "fluentd", host: "", index: "main", event: "a message" },
+      times: 1
+  end
+
+  def test_expand
+    stub_request(:post, "https://localhost:8089/services/collector").
+      to_return(body: '{"text":"Success","code":0}')
+
+    d = create_driver(CONFIG + %[
+      source ${record["source"]}
+      sourcetype ${tag_parts[0]}
+    ])
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({"message" => "a message", "source" => "source-from-record"}, time)
+
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "source-from-record", sourcetype: "test", host: "", index: "main", event: "a message" },
+      times: 1
+  end
+
+  def test_4XX_error_retry
+    stub_request(:post, "https://localhost:8089/services/collector").
+      with(headers: {"Authorization" => "Splunk changeme"}).
+      to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
+
+    d = create_driver
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({ "message" => "1" }, time)
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
+      times: 1
+  end
+
+  def test_5XX_error_retry
+    request_count = 0
+    stub_request(:post, "https://localhost:8089/services/collector").
+      with(headers: {"Authorization" => "Splunk changeme"}).
+      to_return do |request|
+        request_count += 1
+
+        if request_count < 5
+          { body: '{"text":"Internal server error","code":8}', status: 500 }
+        else
+          { body: '{"text":"Success","code":0}', status: 200 }
+        end
+      end
+
+
+    d = create_driver(CONFIG + %[
+      post_retry_max 5
+      post_retry_interval 0.1
+    ])
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({ "message" => "1" }, time)
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
+      times: 5
+  end
+
+  def test_write_splitting
+    stub_request(:post, "https://localhost:8089/services/collector").
+      with(headers: {"Authorization" => "Splunk changeme"}).
+      to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
+
+    # A single msg is ~110 bytes
+    d = create_driver(CONFIG + %[
+      batch_size_limit 250
+    ])
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({"message" => "a" }, time)
+    d.emit({"message" => "b" }, time)
+    d.emit({"message" => "c" }, time)
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body:
+        { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "a" }.to_json +
+        { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "b" }.to_json,
+      times: 1
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "c" }.to_json,
+      times: 1
+    assert_requested :post, "https://localhost:8089/services/collector", times: 2
+  end
+
+  def test_utf8
+    stub_request(:post, "https://localhost:8089/services/collector").
+      with(headers: {"Authorization" => "Splunk changeme"}).
+      to_return(body: '{"text":"Success","code":0}')
+
+    d = create_driver(CONFIG + %[
+      all_items true
+    ])
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: "     f-8", with: ["  ","   ","f-8"]}}},
+      times: 1
+  end
+
+  def test_utf8
+    stub_request(:post, "https://localhost:8089/services/collector").
+      with(headers: {"Authorization" => "Splunk changeme"}).
+      to_return(body: '{"text":"Success","code":0}')
+
+    d = create_driver(CONFIG + %[
+      all_items true
+    ])
+
+    time = Time.parse("2010-01-02 13:14:15 UTC").to_i
+    d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
+    d.run
+
+    assert_requested :post, "https://localhost:8089/services/collector",
+      headers: {"Authorization" => "Splunk changeme"},
+      body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: "     f-8", with: ["  ","   ","f-8"]}}},
+      times: 1
+  end
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-splunk-http-eventcollector-test
+version: !ruby/object:Gem::Version
+  version: 0.3.0.1
+platform: ruby
+authors:
+- Bryce Chidester
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.12
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.9'
+description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
+email:
+- bryce.chidester@calyptix.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- test/helper.rb
+- fluent-plugin-splunk-http-eventcollector-test.gemspec
+- lib/fluent/plugin/out_splunk-http-eventcollector.rb
+- test/plugin/test_out_splunk-http-eventcollector.rb
+homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
+licenses:
+- BSD-2-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: Splunk output plugin for Fluentd with increased batch_ size_limit
+test_files:
+- test/helper.rb
+- test/plugin/test_out_splunk-http-eventcollector.rb