fluent-plugin-splunkhec 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: '0988078609653d3278e1ef55802b037f10ed324b'
+  data.tar.gz: 972ec8325c5b39922faa4a216e7b04a43053f7c5
+SHA512:
+  metadata.gz: a352c9a7087d95048050d81ba319e43f3e13038277568f5649c52f35baec9a40f8c48a4d7092a07c3b5dce009cf6e2c0e1146f60dc8bfb9fe2aa8d39abecd209
+  data.tar.gz: 90ca57989e7b435a6971f24ec3b5f2acfb370762956f3271a7189d3401acc03c6520ea0d72c33bba0051d6766e5928d5836bae6421f79d0ea91a753f9cddd23f

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+## 1.0.0
+
+Added all Splunk HTTP Event Collector field options.
+
+## 0.9.1
+
+Replaced RestClient for net/http.
+
+## 0.9.0
+
+First version

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-googleanalytics.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Coen Meerbeek
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,74 @@
+# fluent-plugin-splunkhec, a plugin for [Fluentd](http://fluentd.org)
+
+## Overview
+
+***Splunk HTTP Event Collector*** output plugin.
+
+Output data from any Fluent input plugin to the Splunk HTTP Event Collector (Splunk HEC).
+
+The Splunk HEC is running on a Heavy Forwarder or single instance. More info about the Splunk HEC architecture in a distributed environment can be found in the Splunk [Docs](http://dev.splunk.com/view/event-collector/SP-CAAAE73)
+
+## Configuration
+
+```config
+<match splunkhec>
+    @type splunkhec
+    host splunk.bluefactory.nl
+    protocol https #optional
+    port 8080 #optional
+    token BAB747F3-744E-41BA
+    index main #optional
+    event_host fluentdhost #optional
+    source fluentd #optional
+    sourcetype data:type #optional
+</source>
+```
+
+## config: host
+
+The host where the Splunk HEC is listening (Heavy Forwarder or Single Instance).
+
+## config: protocol
+
+The protocol on which the Splunk HEC is listening. If you are going to use HTTPS make sure you use a signed certificate. Weak certificates are a work in progress.
+
+## config: port
+
+The port on which the Splunk HEC is listening.
+
+## config: token
+
+Every Splunk HEC requires a token to recieve data. You must configure this insite Splunk [Splunk HEC docs](http://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector).
+Put the token here.
+
+## config: index
+
+The index on the Splunk side to store the data in. Please be aware that the Splunk HTTP Event Collector you've created has the permissions to write to this index. If you don't specify this the plug-in will use "main".
+
+## config: event_host
+
+Specify the host-field for the event data in Splunk. If you don't specify this the plug-in will try to read the hostname running FluentD.
+
+## config: source
+
+Specify the source-field for the event data in Splunk. If you don't specify this the plug-in will use "fluentd".
+
+## config: sourcetype
+
+Specify the sourcetype-field for the event data in Splunk. If you don't specify this the plug-in will use the tag from the FluentD input plug-in.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## TODO
+
+* Add support for SSL verification.
+ 
+## Copyright
+
+Copyright (c) 2017 Coen Meerbeek. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-splunkhec.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-splunkhec"
+  gem.version       = "1.0.0"
+  gem.authors       = "Coen Meerbeek"
+  gem.email         = "cmeerbeek@gmail.com"
+  gem.description   = %q{Output plugin for the Splunk HTTP Event Collector.}
+  gem.homepage      = "https://github.com/cmeerbeek/fluent-plugin-splunkhec"
+  gem.summary       = %q{This plugin allows you to sent events to the Splunk HTTP Event Collector.}
+  
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency "json", '~> 2.0', '>= 2.0.2'
+  gem.add_development_dependency "rake", '~> 0.9', '>= 0.9.2'
+  gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
+  gem.license = 'MIT'
+end

data/lib/fluent/plugin/out_splunkhec.rb

@@ -0,0 +1,122 @@
+require 'fluent/output'
+require 'net/http'
+require 'json'
+
+module Fluent
+  class SplunkHECOutput < BufferedOutput
+    Fluent::Plugin.register_output('splunkhec', self)
+
+    # Primary Splunk HEC configuration parameters
+    config_param :host,     :string, :default => 'localhost', :required => true
+    config_param :protocol, :string, :default => 'http', :required => true
+    config_param :port,     :string, :default => '8088', :required => true
+    config_param :token,    :string, :default => nil, :required => true
+
+    # Splunk event parameters
+    config_param :index,      :string, :default => "main"
+    config_param :event_host, :string, :default => nil
+    config_param :source,     :string, :default => "fluentd"
+    config_param :sourcetype, :string, :default => nil
+
+    # This method is called before starting.
+    # Here we construct the Splunk HEC URL to POST data to
+    # If the configuration is invalid, raise Fluent::ConfigError.
+    def configure(conf)
+      super
+
+      @protocol = conf['protocol']
+
+      @splunk_url = @protocol + '://' + conf['host'] + ':' + conf['port'] + '/services/collector/event'
+      log.debug 'splunkhec: sent data to ' + @splunk_url
+      if conf['token'] != nil
+        @token = conf['token']
+      else
+        raise 'splunkhec: token is empty, please provide a token for this plugin to work'
+      end
+
+      if conf['event_host'] == nil
+        @event_host = `hostname`
+        @event_host = @event_host.delete!("\n")
+      else
+        @event_host = conf['event_host']
+      end
+
+      if conf['sourcetype'] == nil
+        @event_sourcetype = 'tag'
+      else
+        @event_sourcetype = conf['sourcetype']
+      end
+      
+      @event_index = @index
+      @event_source = @source
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    # This method is called when an event reaches to Fluentd.
+    # Use msgpack to serialize the object.
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Loop through all records and sent them to Splunk
+    def write(chunk)
+      begin
+        chunk.msgpack_each {|(tag,time,record)|
+          # Parse record to Splunk event format
+          case record
+          when Fixnum
+            event = record.to_s
+          when Hash
+            event = record.to_json.gsub("\"", %q(\\\"))
+          else
+            event = record
+          end
+
+          if @event_sourcetype == 'tag'
+            @event_sourcetype = tag
+          end
+
+          # Build body for the POST request
+          body = '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + @event_sourcetype + '", "source" :"' + @event_source + '", "index" :"' + @event_index + '", "host" : "' + @event_host + '"}'
+          log.debug "splunkhec: " + body + "\n"
+          
+          uri = URI(@splunk_url)
+          
+          # Create client
+          http = Net::HTTP.new(uri.host, uri.port)
+          
+          # Create Request
+          req =  Net::HTTP::Post.new(uri)
+          # Add headers
+          req.add_field "Authorization", "Splunk #{@token}"
+          # Add headers
+          req.add_field "Content-Type", "application/json; charset=utf-8"
+          # Set body
+          req.body = body
+          # Handle SSL
+          if @protocol == 'https'
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          end
+
+          # Fetch Request
+          res = http.request(req)
+          log.debug "splunkhec: response HTTP Status Code is #{res.code}"
+          if res.code.to_i != 200
+            log.debug "splunkhec: response body is #{res.body}"
+          end
+        }
+      rescue => err
+        log.fatal("splunkhec: caught exception; exiting")
+        log.fatal(err)
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'bundler'
+require 'fluent/input'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require "test/unit"
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require "fluent/test"
+unless ENV.has_key?("VERBOSE")
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      #pass
+    end
+  }
+  $log = nulllogger
+end
+
+require "fluent/plugin/in_splunkhec"
+
+class Test::Unit::TestCase
+end
+

data/test/plugin/test_out_splunkhec.rb

@@ -0,0 +1,28 @@
+require 'helper'
+
+class SplunkHECOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  ### for Splunk HEC
+  CONFIG_SPLUNKHEC = %[
+    host splunk.bluefactory.nl
+    protocol https
+    port 8443
+    token BAB747F3-744E-41BA
+  ]
+
+  def create_driver_ga(conf = CONFIG_SPLUNKHEC)
+    Fluent::Test::InputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
+  end
+
+  def test_configure_splunkhec
+    d = create_driver_splunkhec
+    assert_equal 'splunk.bluefactory.nl', d.instance.host
+    assert_equal 'https' , d.instance.protocol
+    assert_equal '8443' , d.instance.port
+    assert_equal 'BAB747F3-744E-41BA', d.instance.token
+  end
+
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-splunkhec
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Coen Meerbeek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: Output plugin for the Splunk HTTP Event Collector.
+email: cmeerbeek@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-splunkhec.gemspec
+- lib/fluent/plugin/out_splunkhec.rb
+- test/helper.rb
+- test/plugin/test_out_splunkhec.rb
+homepage: https://github.com/cmeerbeek/fluent-plugin-splunkhec
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: This plugin allows you to sent events to the Splunk HTTP Event Collector.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_splunkhec.rb