fluent-plugin-splunk-http-eventcollector 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a9bb8b2d4ecc9e67d89c91b7c146b3be7396e12d
-  data.tar.gz: 08df84dc5e71141a4558fe2424798062061283dd
+  metadata.gz: 6af449c8480d06db46282037eecc373581957efe
+  data.tar.gz: 6372e889d37f0c8ec98f540e53e354a5673dcf85
 SHA512:
-  metadata.gz: e51aabb236cd36f73b031c08982741defdc91abc27add9274671678bfdf2ed7ef06db4ad76feb69108d785a39a731681020ecc08dc49fe106c15054190d76844
-  data.tar.gz: b3196810b72dde78988d46495b9447dd9669851135beb8a5d3912fa1cb26fd179cc07f7fc51fa0a370514f19d61905b9c8b039c7779caf9de380c1152140a9de
+  metadata.gz: 3d647ecdfcfdcf530b494d4306c7c5df8eec958ea53d806de94073213bc7fe59b1fd92323db598d4bd9721a374f679ad2ed292a9724f28a19767e7345e0a36dc
+  data.tar.gz: e11b0d0fbba97638f5614a4dd3aa1e57b7d116af75ebed13a9e54fa9dc95aee19ee5f9d46256230085c1636644ba084c56c5545bf009f2f7c81498189ce13b47

data/README.md

@@ -46,6 +46,10 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-splunk-http-eventcollector
 
+Whatever is appropriate for your environment. Note: If you're using the
+`td-agent` package, it brings with it its own "embedded" Ruby environment with
+either `td-agent-gem` or `/opt/td-agent/embedded/bin/gem` depending on platform.
+
 ## Configuration
 
 Put the following lines to your fluent.conf:
@@ -57,6 +61,10 @@ Put the following lines to your fluent.conf:
       # default: localhost:8088
       server localhost:8088
 
+      # protocol: Connect to Splunk server via 'http' or 'https'
+      # default: https
+      #protocol http
+
       # verify: SSL server verification
       # default: true
       #verify false

data/fluent-plugin-splunk-http-eventcollector.gemspec

@@ -3,13 +3,13 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name             = "fluent-plugin-splunk-http-eventcollector"
-  gem.version          = "0.0.4"
+  gem.version          = "0.1.0"
   gem.authors          = ["Bryce Chidester"]
   gem.email            = ["bryce.chidester@calyptix.com"]
   gem.summary          = "Splunk output plugin for Fluentd"
   gem.description      = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
   gem.homepage         = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
-  gem.license          = 'BSD 2-clause'
+  gem.license          = 'BSD-2-Clause'
   gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
   gem.files            = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
                            "Rakefile", "test/helper.rb",
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit", '~> 3.1'
   gem.add_runtime_dependency "fluentd", '~> 0.12'
   gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
+  gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name", '~> 0.1.0'
 end

data/lib/fluent/plugin/out_splunk-http-eventcollector.rb

@@ -25,11 +25,16 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 =end
+require 'fluent/mixin/rewrite_tag_name'
 
 module Fluent
 class SplunkHTTPEventcollectorOutput < BufferedOutput
-  Plugin.register_output('splunk-http-eventcollector', self)
   
+  Plugin.register_output('splunk-http-eventcollector', self)
+
+  include Fluent::HandleTagNameMixin
+  include Fluent::Mixin::RewriteTagName
+
   config_param :test_mode, :bool, :default => false
   
   config_param :server, :string, :default => 'localhost:8088'
@@ -37,9 +42,13 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   config_param :token, :string, :default => nil
   
   # Event parameters
+  config_param :protocol, :string, :default => 'https'
   config_param :host, :string, :default => nil
   config_param :index, :string, :default => 'main'
+  config_param :all_items, :bool, :default => false
   
+  config_param :sourcetype, :string, :default => 'fluentd'
+  config_param :source, :string, :default => nil
   config_param :post_retry_max, :integer, :default => 5
   config_param :post_retry_interval, :integer, :default => 5
   
@@ -50,7 +59,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   # Called on class load (class initializer)
   def initialize
     super
-    $log.debug "splunk-http-eventcollector(initialize) called"
+    log.trace "splunk-http-eventcollector(initialize) called"
     require 'net/http/persistent'
     require 'openssl'
   end  # initialize
@@ -60,9 +69,9 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   ## If the configuration is invalid, raise Fluent::ConfigError.
   def configure(conf)
     super
-    $log.debug "splunk-http-eventcollector(configure) called"
+    log.trace "splunk-http-eventcollector(configure) called"
     begin
-      @splunk_uri = URI "https://#{@server}/services/collector"
+      @splunk_uri = URI "#{@protocol}://#{@server}/services/collector"
     rescue
       raise ConfigError, "Unable to parse the server into a URI."
     end
@@ -73,41 +82,49 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   ## Open sockets or files here.
   def start
     super
-    $log.debug "splunk-http-eventcollector(start) called"
+    log.trace "splunk-http-eventcollector(start) called"
     @http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
     @http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
     @http.override_headers['Content-Type'] = 'application/json'
     @http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
     @http.override_headers['Authorization'] = "Splunk #{@token}"
     
-    $log.debug "initialized for splunk-http-eventcollector"
+    log.trace "initialized for splunk-http-eventcollector"
   end
   
   ## This method is called when shutting down.
   ## Shutdown the thread and close sockets or files here.
   def shutdown
     super
-    $log.debug "splunk-http-eventcollector(shutdown) called"
+    log.trace "splunk-http-eventcollector(shutdown) called"
     
     @http.shutdown
-    $log.debug "shutdown from splunk-http-eventcollector"
+    log.trace "shutdown from splunk-http-eventcollector"
   end  # shutdown
   
   ## This method is called when an event reaches to Fluentd. (like unbuffered emit())
   ## Convert the event to a raw string.
   def format(tag, time, record)
-    #$log.debug "splunk-http-eventcollector(format) called"
+    #log.trace "splunk-http-eventcollector(format) called"
     # Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
+    @placeholder_expander.set_tag(tag)
+
     splunk_object = Hash[
-        "event" => record["message"],
         "time" => time.to_i,
-        "source" => tag.to_s,
+        "source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source) end,
+        "sourcetype" => @placeholder_expander.expand(@sourcetype.to_s),
         "host" => @host.to_s,
-        "index" => @index.to_s
-        ]
+        "index" =>  @placeholder_expander.expand(@index)
+      ]
+    # TODO: parse different source types as expected: KVP, JSON, TEXT
+    if @all_items
+        splunk_object["event"] = record
+    else
+        splunk_object["event"] = record["message"]
+    end
     json_event = splunk_object.to_json
-    #$log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
-    #$log.debug "format: returning: #{[tag, record].to_json.to_s}"
+    #log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
+    #log.debug "format: returning: #{[tag, record].to_json.to_s}"
     json_event
   end
   
@@ -122,7 +139,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   ##
   ## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
   def write(chunk)
-    $log.debug "splunk-http-eventcollector(write) called"
+    log.trace "splunk-http-eventcollector(write) called"
     
     # Break the concatenated string of JSON-formatted events into an Array
     split_chunk = chunk.read.split("}{").each do |x|
@@ -130,33 +147,33 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
       x.prepend("{") unless x.start_with?("{")
       x << "}" unless x.end_with?("}")
     end
-    $log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
+    log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
         "#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
     # If fluentd is pushing too much data to Splunk at once, split up the payload
     # Don't care about the number of events so much as the POST size (bytes)
     #if split_chunk.size > @batch_event_limit
-    #  $log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
+    #  log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
     #      "events in a single push to Splunk. The configured limit is " + 
     #      "#{numfmt(@batch_event_limit)}."
     #end
     if chunk.read.bytesize > @batch_size_limit
-      $log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
+      log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
           "bytes in a single push to Splunk. The configured limit is " + 
           "#{numfmt(@batch_size_limit)} bytes."
       newbuffer = Array.new
       split_chunk_counter = 0
       split_chunk.each do |c|
         split_chunk_counter = split_chunk_counter + 1
-        #$log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
+        #log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
         #    "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
         #    "c.bytesize=#{numfmt(c.bytesize)} ????"
         if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
-          #$log.debug "Appended!"
+          #log.debug "Appended!"
           newbuffer << c
         else
           # Reached the limit - push the current newbuffer.join, and reset
-          #$log.debug "Would exceed limit. Flushing newbuffer and continuing."
-          $log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
+          #log.debug "Would exceed limit. Flushing newbuffer and continuing."
+          log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
               "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
               "c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
               "flushing current buffer to Splunk."
@@ -175,15 +192,15 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
   def push_buffer(body)
     post = Net::HTTP::Post.new @splunk_uri.request_uri
     post.body = body
-    $log.debug "POST #{@splunk_uri}"
+    log.debug "POST #{@splunk_uri}"
     if @test_mode
-      $log.debug "TEST_MODE Payload: #{body}"
+      log.debug "TEST_MODE Payload: #{body}"
       return
     end
     # retry up to :post_retry_max times
     1.upto(@post_retry_max) do |c|
       response = @http.request @splunk_uri, post
-      $log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
+      log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
           "(#{response.message})"
       # TODO check the actual server response too (it's JSON)
       if response.code == "200"  # and...
@@ -192,18 +209,18 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
       # TODO check 40X response within post_retry_max and retry
       elsif response.code.match(/^50/) and c < @post_retry_max
         # retry
-        $log.warn "#{@splunk_uri}: Server error #{response.code} (" +
+        log.warn "#{@splunk_uri}: Server error #{response.code} (" +
             "#{response.message}). Retrying in #{@post_retry_interval} " +
             "seconds.\n#{response.body}"
         sleep @post_retry_interval
         next
       elsif response.code.match(/^40/)
         # user error
-        $log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
+        log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
         break
       elsif c < @post_retry_max
         # retry
-        $log.debug "#{@splunk_uri}: Retrying..."
+        log.debug "#{@splunk_uri}: Retrying..."
         sleep @post_retry_interval
         next
       else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-http-eventcollector
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Bryce Chidester
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-04 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
 email:
 - bryce.chidester@calyptix.com
@@ -72,7 +86,7 @@ files:
 - test/plugin/test_out_splunk-http-eventcollector.rb
 homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
 licenses:
-- BSD 2-clause
+- BSD-2-Clause
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -90,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Splunk output plugin for Fluentd