fluent-plugin-splunkhec 1.1 → 1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b095141e9e4ae6cecb11bfc80c7fe26503858909
-  data.tar.gz: e080ff03c7f54382761dfac6957807be0b278106
+  metadata.gz: b9180a1fe84fc9e92e0bd1cc7f190030fc66ac82
+  data.tar.gz: 14530543bf7ad149b53c698ffde814717fa87ffa
 SHA512:
-  metadata.gz: 1cbaa497638b467481b56f0472eb6eb20067784e88d6213baa8b74729194baa9fcfb5941f9e14fff3937d4d29ceed282dc36254810dc619d508cc599fe1eb54c
-  data.tar.gz: 8f5046116bd7fa08500b7b7b6ff0814cb46dd8b4b8ce269879cc6acc75f29ead9f64531972e1b5abad2c4a05d113e634d75fc74701529e087b7bbe2fe2c8fd22
+  metadata.gz: 6d9c296da32fa4c7c5905e62d912a9459fbdefb5f513c92b19caac58f08630ebba80552f6d36474359edfb380c8093d0adb5e43d1d532fda335cab7757af70ba
+  data.tar.gz: b473a7be07fd74d4f50ab81218a8e9dc07cfb598f70637fdb9e5a0c25d8361c0ee199382b90af3964dd097e281e311dd21df2f150ee670849935854324df7155

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.2
+
+- Improved unit test coverage
+- Removed superfluous instance variables
+- Added feature send_batched_events
+- Rescue hosts that do not have hostname command installed.
+
 ## 1.1
 
 - Added send_event_as_json parameter to sent real json

data/README.md

@@ -23,6 +23,7 @@ The Splunk HEC is running on a Heavy Forwarder or single instance. More info abo
     sourcetype data:type #optional
     usejson true #optional defaults to true
     send_event_as_json true #optional
+    send_batched_events false #optional
 </source>
 ```
 
@@ -67,6 +68,10 @@ Specify if an event should be sent as json rather than as a string. Can be 'true
 
 Specify the event type as JSON (true|default) or raw (false) for sending Log4J messages so Splunk so it can parse the time field it self based on the format 'time' regex match found in the source, uses millisecond precision.
 
+## config: send_batched_events
+
+Specify that all events in a FluentD chunk should be sent in batch to Splunk. Defaults to 'false' which sends one event at a time. Batching events will reduce the load on the Splunk HEC. Max chunk size is controlled by config parameter 'buffer_chunk_limit' and should be matched by the Splunk limit 'max_content_length'. Please see this [blog post](https://www.splunk.com/blog/2016/08/12/handling-http-event-collector-hec-content-length-too-large-errors-without-pulling-your-hair-out.html) for details.
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-splunkhec.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-splunkhec"
-  gem.version       = "1.1"
+  gem.version       = "1.2"
   gem.authors       = "Coen Meerbeek"
   gem.email         = "cmeerbeek@gmail.com"
   gem.description   = %q{Output plugin for the Splunk HTTP Event Collector.}
@@ -20,5 +20,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency "json", '~> 2.0', '>= 2.0.2'
   gem.add_development_dependency "rake", '~> 0.9', '>= 0.9.2'
   gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
+  gem.add_development_dependency "webmock", '>= 3.0'
   gem.license = 'MIT'
 end

data/lib/fluent/plugin/out_splunkhec.rb

@@ -10,51 +10,32 @@ module Fluent
     config_param :host,     :string, :default => 'localhost'
     config_param :protocol, :string, :default => 'http'
     config_param :port,     :string, :default => '8088'
-    config_param :token,    :string, :default => nil
+    config_param :token,    :string
 
     # Splunk event parameters
-    config_param :index,      :string, :default => "main"
-    config_param :event_host, :string, :default => nil
-    config_param :source,     :string, :default => "fluentd"
-    config_param :sourcetype, :string, :default => nil
-    config_param :send_event_as_json, :string, :default => "false"
-    config_param :usejson,    :string, :default => "true"
+    config_param :index,               :string, :default => 'main'
+    config_param :event_host,          :string, :default => nil
+    config_param :source,              :string, :default => 'fluentd'
+    config_param :sourcetype,          :string, :default => 'tag'
+    config_param :send_event_as_json,  :bool,   :default => false
+    config_param :usejson,             :bool,   :default => true
+    config_param :send_batched_events, :bool,   :default => false
 
     # This method is called before starting.
     # Here we construct the Splunk HEC URL to POST data to
     # If the configuration is invalid, raise Fluent::ConfigError.
     def configure(conf)
       super
-
       @splunk_url = @protocol + '://' + @host + ':' + @port + '/services/collector/event'
       log.debug 'splunkhec: sent data to ' + @splunk_url
-      if conf['token'] != nil
-        @token = conf['token']
-      else
-        raise 'splunkhec: token is empty, please provide a token for this plugin to work'
-      end
 
       if conf['event_host'] == nil
-        @event_host = `hostname`
-        @event_host = @event_host.delete!("\n")
-      else
-        @event_host = conf['event_host']
-      end
-
-      if conf['sourcetype'] == nil
-        @event_sourcetype = 'tag'
-      else
-        @event_sourcetype = conf['sourcetype']
+        begin
+          @event_host = `hostname`.delete!("\n")
+        rescue
+          @event_host = 'unknown'
+        end
       end
-
-      if conf['send_event_as_json'] == 'true'
-        @event_send_as_json = true
-      else
-        @event_send_as_json = false
-      end
-
-      @event_index = @index
-      @event_source = @source
     end
 
     def start
@@ -74,13 +55,14 @@ module Fluent
     # Loop through all records and sent them to Splunk
     def write(chunk)
       begin
+        body = ''
         chunk.msgpack_each {|(tag,time,record)|
           # Parse record to Splunk event format
           case record
           when Fixnum
             event = record.to_s
           when Hash
-            if @event_send_as_json
+            if @send_event_as_json
               event = record.to_json
             else
               event = record.to_json.gsub("\"", %q(\\\"))
@@ -89,51 +71,63 @@ module Fluent
             event = record
           end
 
-          if @event_sourcetype == 'tag'
-            @event_sourcetype = tag
-          end
+          sourcetype = @sourcetype == 'tag' ? tag : @sourcetype
 
           # Build body for the POST request
-          if @usejson == 'false'
+          if !@usejson
             event = record["time"]+ " " + record["message"].to_json.gsub(/^"|"$/,"")
-            body = '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + @event_sourcetype + '", "source" :"' + @event_source + '", "index" :"' + @event_index + '", "host" : "' + @event_host + '"}'
-          elsif @event_send_as_json
-            body = '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + @event_sourcetype + '", "source" :"' + @event_source + '", "index" :"' + @event_index + '", "host" : "' + @event_host + '"}'
+            body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
+          elsif @send_event_as_json
+            body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
           else
-            body = '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + @event_sourcetype + '", "source" :"' + @event_source + '", "index" :"' + @event_index + '", "host" : "' + @event_host + '"}'
-          end
-          log.debug "splunkhec: " + body + "\n"
-          
-          uri = URI(@splunk_url)
-          
-          # Create client
-          http = Net::HTTP.new(uri.host, uri.port)
-          
-          # Create Request
-          req =  Net::HTTP::Post.new(uri)
-          # Add headers
-          req.add_field "Authorization", "Splunk #{@token}"
-          # Add headers
-          req.add_field "Content-Type", "application/json; charset=utf-8"
-          # Set body
-          req.body = body
-          # Handle SSL
-          if @protocol == 'https'
-            http.use_ssl = true
-            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
           end
 
-          # Fetch Request
-          res = http.request(req)
-          log.debug "splunkhec: response HTTP Status Code is #{res.code}"
-          if res.code.to_i != 200
-            log.debug "splunkhec: response body is #{res.body}"
+          if @send_batched_events
+            body << "\n"
+          else
+            send_to_splunk(body)
+            body = ''
           end
         }
+
+        if @send_batched_events
+          send_to_splunk(body)
+        end
       rescue => err
         log.fatal("splunkhec: caught exception; exiting")
         log.fatal(err)
       end
     end
+
+    def send_to_splunk(body)
+      log.debug "splunkhec: " + body + "\n"
+
+      uri = URI(@splunk_url)
+
+      # Create client
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      # Create Request
+      req = Net::HTTP::Post.new(uri)
+      # Add headers
+      req.add_field "Authorization", "Splunk #{@token}"
+      # Add headers
+      req.add_field "Content-Type", "application/json; charset=utf-8"
+      # Set body
+      req.body = body
+      # Handle SSL
+      if @protocol == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      # Fetch Request
+      res = http.request(req)
+      log.debug "splunkhec: response HTTP Status Code is #{res.code}"
+      if res.code.to_i != 200
+        log.debug "splunkhec: response body is #{res.body}"
+      end
+    end
   end
 end

data/test/plugin/test_out_splunkhec.rb

@@ -1,28 +1,165 @@
 require 'helper'
+require 'webmock/test_unit'
+
 
 class SplunkHECOutputTest < Test::Unit::TestCase
+  HOST = 'splunk.example.com'
+  PROTOCOL = 'https'
+  PORT = '8443'
+  TOKEN = 'BAB747F3-744E-41BA'
+  SOURCE = 'fluentd'
+  INDEX = 'main'
+  EVENT_HOST = 'some_host'
+  SOURCETYPE = 'log'
+
+  SPLUNK_URL = "#{PROTOCOL}://#{HOST}:#{PORT}/services/collector/event"
+
+  ### for Splunk HEC
+  CONFIG = %[
+    host #{HOST}
+    protocol #{PROTOCOL}
+    port #{PORT}
+    token #{TOKEN}
+    source #{SOURCE}
+    index #{INDEX}
+    event_host #{EVENT_HOST}
+  ]
+
+  def create_driver_splunkhec(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
+  end
+
   def setup
     Fluent::Test.setup
+    require 'fluent/plugin/out_splunkhec'
+    stub_request(:any, SPLUNK_URL)
   end
 
-  ### for Splunk HEC
-  CONFIG_SPLUNKHEC = %[
-    host splunk.bluefactory.nl
-    protocol https
-    port 8443
-    token BAB747F3-744E-41BA
-  ]
+  def test_should_require_mandatory_parameter_token
+    assert_raise Fluent::ConfigError do
+      create_driver_splunkhec(%[])
+    end
+  end
 
-  def create_driver_splunkhec(conf = CONFIG_SPLUNKHEC)
-    Fluent::Test::InputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
+  def test_should_use_default_values_for_optional_parameters
+    d = create_driver_splunkhec(%[token some_token])
+    assert_equal 'localhost', d.instance.host
+    assert_equal 'http', d.instance.protocol
+    assert_equal '8088', d.instance.port
+    assert_equal 'main', d.instance.index
+    assert_equal `hostname`.delete!("\n"), d.instance.event_host
+    assert_equal 'fluentd', d.instance.source
+    assert_equal 'tag', d.instance.sourcetype
+    assert_equal false, d.instance.send_event_as_json
+    assert_equal true, d.instance.usejson
+    assert_equal false, d.instance.send_batched_events
+    assert_equal 'some_token', d.instance.token
   end
 
-  def test_configure_splunkhec
+  def test_should_configure_splunkhec
     d = create_driver_splunkhec
-    assert_equal 'splunk.bluefactory.nl', d.instance.host
-    assert_equal 'https' , d.instance.protocol
-    assert_equal '8443' , d.instance.port
-    assert_equal 'BAB747F3-744E-41BA', d.instance.token
+    assert_equal HOST, d.instance.host
+    assert_equal PROTOCOL, d.instance.protocol
+    assert_equal PORT, d.instance.port
+    assert_equal TOKEN, d.instance.token
+  end
+
+  def test_should_post_formatted_event_to_splunk
+    sourcetype = 'log'
+    time = 123456
+    record = {'message' => 'data'}
+
+    splunk_request = stub_request(:post, SPLUNK_URL)
+                         .with(
+                             headers: {
+                                 'Authorization' => "Splunk #{TOKEN}",
+                                 'Content-Type' => 'application/json; charset=utf-8'
+                             },
+                             body: {
+                                 'time' => time,
+                                 'event' => record.to_json,
+                                 'sourcetype' => sourcetype,
+                                 'source' => SOURCE,
+                                 'index' => INDEX,
+                                 'host' => EVENT_HOST
+                             })
+
+    d = create_driver_splunkhec(CONFIG + %[sourcetype #{sourcetype}])
+    d.run do
+      d.emit(record, time)
+    end
+
+    assert_requested(splunk_request)
+  end
+
+  def test_should_use_tag_as_sourcetype_when_configured
+    splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'sourcetype' => 'test'}))
+
+    d = create_driver_splunkhec(CONFIG + %[sourcetype tag])
+    d.run do
+      d.emit({'message' => 'data'}, 123456)
+    end
+
+    assert_requested(splunk_request)
+  end
+
+  def test_should_send_event_as_string_as_default
+    record = {'message' => 'data'}
+    splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record.to_json}))
+
+    d = create_driver_splunkhec(CONFIG + %[send_event_as_json false])
+    d.run do
+      d.emit(record)
+    end
+
+    assert_requested(splunk_request)
+  end
+
+  def test_should_send_event_as_log4j_format_when_configured
+    log_time = '2017-07-02 20:52:39'
+    log_time_millis = '1499028759000'
+    log_event = 'data'
+
+    splunk_request = stub_request(:post, SPLUNK_URL)
+                         .with(body: hash_including({'time' => log_time_millis, 'event' => "#{log_time} #{log_event}"}))
+
+    d = create_driver_splunkhec(CONFIG + %[usejson false])
+    d.run do
+      d.emit({'time' => log_time, 'message' => log_event})
+    end
+
+    assert_requested(splunk_request)
+  end
+
+  def test_should_send_event_as_json_when_configured
+    record = {'message' => 'data'}
+
+    splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record}))
+
+    d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
+    d.run do
+      d.emit(record)
+    end
+
+    assert_requested(splunk_request)
+  end
+
+  def test_should_batch_post_all_events_in_chunk_when_configured
+    record1 = {'message' => 'data'}
+    record2 = {'message' => 'more data'}
+
+    splunk_request = stub_request(:post, SPLUNK_URL).with(body: /\"event\" :#{record1.to_json}.*\"event\" :#{record2.to_json}/m)
+
+    d = create_driver_splunkhec(CONFIG + %[
+      send_event_as_json true
+      send_batched_events true])
+
+    d.run do
+      d.emit(record1)
+      d.emit(record2)
+    end
+
+    assert_requested(splunk_request)
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunkhec
 version: !ruby/object:Gem::Version
-  version: '1.1'
+  version: '1.2'
 platform: ruby
 authors:
 - Coen Meerbeek
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -90,6 +90,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
 description: Output plugin for the Splunk HTTP Event Collector.
 email: cmeerbeek@gmail.com
 executables: []