fluent-plugin-splunkhec 1.7 → 2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64bafb9078216f8761e88b562441c73861fa0b20
-  data.tar.gz: 79933b686c1619ac6ddd79789df0034fd2ec8403
+  metadata.gz: '058b51ece316507dd70cc84bd3ec0c32372e1f8a'
+  data.tar.gz: bff476df06f9a86433c9b10f74975a5b3d69c301
 SHA512:
-  metadata.gz: 89d43832331ea520882e0129cddaea6942a0cc1c9453d3a4ea6997d174f47791f86663797c8966cdcf1785b370354f12945ead0933c02fbaedda34c551cefa0b
-  data.tar.gz: 9ffe57c6011cdf2a349695a05534513c2d216eba7e984a6d980ab43a63cfd79825bf48ed11877aa3d265d18a3abe48435faba86d6ccbe222245885d21c197545
+  metadata.gz: ced5a57eeb88c6c9ea26bc57db6392f25abde00c89f85539b2a5fad3533972816ba35a4d90b22b3dde14b3b391b10a07ff0ae02782de5c783c5dccd092c74a79
+  data.tar.gz: 150458e3fa0c0b39cab8cc9f4c84bf231f45e70cdbe7344a9d610450066595d16ccdaecaa0dc74e133edcdf17a0d65a915458ce5adc2f4d3265154f187aa1599

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 2.0
+
+Migrate to use FluentD v1 API. It doesn't support backwards compatibility.
+
+## 1.9
+
+Reverted source and sourcetpye settings. They now reflect the README.
+
+## 1.8
+
+- Add expand function used in ES plugin an get variables from kubernetes tags in fluent.conf
+
 ## 1.7
 
 - Fixed HTTP request (removed verify none)
@@ -49,4 +61,4 @@ Replaced RestClient for net/http.
 
 ## 0.9.0
 
-First version
+First version

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2017 Coen Meerbeek
+Copyright (c) 2018 Coen Meerbeek
 
 MIT License
 

data/README.md

@@ -86,4 +86,4 @@ Specify that all events in a FluentD chunk should be sent in batch to Splunk. De
  
 ## Copyright
 
-Copyright (c) 2017 Coen Meerbeek. See [LICENSE](LICENSE) for details.
+Copyright (c) 2018 Coen Meerbeek. See [LICENSE](LICENSE) for details.

data/fluent-plugin-splunkhec.gemspec

@@ -4,19 +4,19 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-splunkhec"
-  gem.version       = "1.7"
+  gem.version       = "2.0"
   gem.authors       = "Coen Meerbeek"
   gem.email         = "cmeerbeek@gmail.com"
   gem.description   = %q{Output plugin for the Splunk HTTP Event Collector.}
   gem.homepage      = "https://github.com/cmeerbeek/fluent-plugin-splunkhec"
   gem.summary       = %q{This plugin allows you to sent events to the Splunk HTTP Event Collector.}
-  
+
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+
+  gem.add_dependency "fluentd", [">= 0.14.15", "< 2"]
   gem.add_dependency "yajl-ruby", '>= 1.3.0'
   gem.add_development_dependency "rake", '~> 0.9', '>= 0.9.2'
   gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'

data/lib/fluent/plugin/out_splunkhec.rb

@@ -1,11 +1,15 @@
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'net/http'
 require 'yajl/json_gem'
 
-module Fluent
-  class SplunkHECOutput < BufferedOutput
+module Fluent::Plugin
+  class SplunkHECOutput < Output
     Fluent::Plugin.register_output('splunkhec', self)
 
+    helpers :compat_parameters, :event_emitter
+
+    DEFAULT_BUFFER_TYPE = "memory"
+
     # Primary Splunk HEC configuration parameters
     config_param :host,     :string, :default => 'localhost'
     config_param :protocol, :string, :default => 'http'
@@ -15,19 +19,24 @@ module Fluent
     # Splunk event parameters
     config_param :index,               :string, :default => 'main'
     config_param :event_host,          :string, :default => nil
-    config_param :source,              :string, :default => 'tag'
-    config_param :sourcetype,          :string, :default => 'fluentd'
+    config_param :source,              :string, :default => 'fluentd'
+    config_param :sourcetype,          :string, :default => 'tag'
     config_param :send_event_as_json,  :bool,   :default => false
     config_param :usejson,             :bool,   :default => true
     config_param :send_batched_events, :bool,   :default => false
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
     # This method is called before starting.
     # Here we construct the Splunk HEC URL to POST data to
     # If the configuration is invalid, raise Fluent::ConfigError.
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
       @splunk_url = @protocol + '://' + @host + ':' + @port + '/services/collector/event'
-      log.info 'splunkhec: sent data to ' + @splunk_url
+      log.info 'splunkhec: sending data to ' + @splunk_url
 
       if conf['event_host'] == nil
         begin
@@ -36,6 +45,7 @@ module Fluent
           @event_host = 'unknown'
         end
       end
+      @packer = Fluent::Engine.msgpack_factory.packer
     end
 
     def start
@@ -46,16 +56,67 @@ module Fluent
       super
     end
 
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     # This method is called when an event reaches to Fluentd.
     # Use msgpack to serialize the object.
     def format(tag, time, record)
-      [tag, time, record].to_msgpack
+      @packer.pack([tag, time, record]).to_s
+    end
+
+    def expand_param(param, tag, time, record)
+      # check for '${ ... }'
+      #   yes => `eval`
+      #   no  => return param
+      return param if (param =~ /\${.+}/).nil?
+
+      # check for 'tag_parts[]'
+      # separated by a delimiter (default '.')
+      tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
+
+      # pull out section between ${} then eval
+      inner = param.clone
+      while inner.match(/\${.+}/)
+        to_eval = inner.match(/\${(.+?)}/){$1}
+
+        if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
+          return to_eval
+        elsif !(to_eval =~/tag_parts\[.+\]/).nil? && tag_parts.nil?
+          return to_eval
+        elsif !(to_eval =~/time/).nil? && time.nil?
+          return to_eval
+        else
+          inner.sub!(/\${.+?}/, eval( to_eval ))
+        end
+      end
+      inner
     end
 
     # Loop through all records and sent them to Splunk
     def write(chunk)
       body = ''
       chunk.msgpack_each {|(tag,time,record)|
+
+        # define index and sourcetype dynamically
+        begin
+          index = expand_param(@index, tag, time, record)
+          sourcetype = expand_param(@sourcetype, tag, time, record)
+          event_host = expand_param(@event_host, tag, time, record)
+          token = expand_param(@token, tag, time, record)
+        rescue => e
+          # handle dynamic parameters misconfigurations
+          router.emit_error_event(tag, time, record, e)
+          next
+        end
+        log.debug "routing event from #{event_host} to #{index} index"
+        log.debug "expanded token #{token}"
+
         # Parse record to Splunk event format
         case record
         when Integer
@@ -70,41 +131,42 @@ module Fluent
           event = record
         end
 
-        source = @source == 'tag' ? tag : @source
+        sourcetype = @sourcetype == 'tag' ? tag : @sourcetype
 
         # Build body for the POST request
         if !@usejson
           event = record["time"]+ " " + record["message"].to_json.gsub(/^"|"$/,"")
-          body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
+          body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
         elsif @send_event_as_json
-          body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
+          body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
         else
-          body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
+          body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
         end
 
         if @send_batched_events
           body << "\n"
         else
-          send_to_splunk(body)
+          send_to_splunk(body, token)
           body = ''
         end
       }
 
       if @send_batched_events
-        send_to_splunk(body)
+        send_to_splunk(body, token)
       end
     end
 
-    def send_to_splunk(body)
+    def send_to_splunk(body, token)
       log.debug "splunkhec: " + body + "\n"
 
       uri = URI(@splunk_url)
 
       # Create client
       http = Net::HTTP.new(uri.host, uri.port)
+      http.set_debug_output(log.debug)
 
       # Create request
-      req = Net::HTTP::Post.new(uri, "Content-Type" => "application/json; charset=utf-8", "Authorization" => "Splunk #{@token}")
+      req = Net::HTTP::Post.new(uri, "Content-Type" => "application/json; charset=utf-8", "Authorization" => "Splunk #{token}")
       req.body = body
 
       # Handle SSL

data/test/helper.rb

@@ -14,18 +14,9 @@ require "test/unit"
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require "fluent/test"
-unless ENV.has_key?("VERBOSE")
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      #pass
-    end
-  }
-  $log = nulllogger
-end
+require "fluent/test/driver/output"
 
 require "fluent/plugin/out_splunkhec"
 
 class Test::Unit::TestCase
 end
-

data/test/plugin/test_out_splunkhec.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'helper'
 require 'webmock/test_unit'
 
@@ -26,7 +27,7 @@ class SplunkHECOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver_splunkhec(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHECOutput).configure(conf)
   end
 
   def setup
@@ -48,8 +49,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
     assert_equal '8088', d.instance.port
     assert_equal 'main', d.instance.index
     assert_equal `hostname`.delete!("\n"), d.instance.event_host
-    assert_equal 'fluentd', d.instance.sourcetype
-    assert_equal 'tag', d.instance.source
+    assert_equal 'tag', d.instance.sourcetype
+    assert_equal 'fluentd', d.instance.source
     assert_equal false, d.instance.send_event_as_json
     assert_equal true, d.instance.usejson
     assert_equal false, d.instance.send_batched_events
@@ -85,8 +86,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
                              })
 
     d = create_driver_splunkhec(CONFIG + %[sourcetype #{sourcetype}])
-    d.run do
-      d.emit(record, time)
+    d.run(default_tag: 'test')  do
+      d.feed(time, record)
     end
 
     assert_requested(splunk_request)
@@ -96,8 +97,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
     splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'sourcetype' => 'test'}))
 
     d = create_driver_splunkhec(CONFIG + %[sourcetype test])
-    d.run do
-      d.emit({'message' => 'data'}, 123456)
+    d.run(default_tag: 'test') do
+      d.feed(123456, {'message' => 'data'})
     end
 
     assert_requested(splunk_request)
@@ -108,8 +109,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
     splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record.to_json}))
 
     d = create_driver_splunkhec(CONFIG + %[send_event_as_json false])
-    d.run do
-      d.emit(record)
+    d.run(default_tag: 'test') do
+      d.feed(record)
     end
 
     assert_requested(splunk_request)
@@ -124,8 +125,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
                          .with(body: hash_including({'time' => log_time_millis, 'event' => "#{log_time} #{log_event}"}))
 
     d = create_driver_splunkhec(CONFIG + %[usejson false])
-    d.run do
-      d.emit({'time' => log_time, 'message' => log_event})
+    d.run(default_tag: 'test') do
+      d.feed({'time' => log_time, 'message' => log_event})
     end
 
     assert_requested(splunk_request)
@@ -137,8 +138,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
     splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record}))
 
     d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
-    d.run do
-      d.emit(record)
+    d.run(default_tag: 'test') do
+      d.feed(record)
     end
 
     assert_requested(splunk_request)
@@ -154,9 +155,9 @@ class SplunkHECOutputTest < Test::Unit::TestCase
       send_event_as_json true
       send_batched_events true])
 
-    d.run do
-      d.emit(record1)
-      d.emit(record2)
+    d.run(default_tag: 'test') do
+      d.feed(record1)
+      d.feed(record2)
     end
 
     assert_requested(splunk_request)
@@ -165,10 +166,10 @@ class SplunkHECOutputTest < Test::Unit::TestCase
   def test_should_raise_exception_when_splunk_returns_error_to_make_fluentd_retry_later
     stub_request(:any, SPLUNK_URL).to_return(status: 403, body: {'text' => 'Token disabled', 'code' => 1}.to_json)
 
-    assert_raise Fluent::SplunkHECOutputError do
+    assert_raise Fluent::Plugin::SplunkHECOutputError do
       d = create_driver_splunkhec
-      d.run do
-        d.emit({'message' => 'data'})
+      d.run(default_tag: 'test') do
+        d.feed({'message' => 'data'})
       end
     end
   end
@@ -179,8 +180,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
     splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => {'message' => '©2017'}}))
 
     d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
-    d.run do
-      d.emit(record)
+    d.run(default_tag: 'test') do
+      d.feed(record)
     end
 
     assert_requested(splunk_request)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunkhec
 version: !ruby/object:Gem::Version
-  version: '1.7'
+  version: '2.0'
 platform: ruby
 authors:
 - Coen Meerbeek
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-15 00:00:00.000000000 Z
+date: 2019-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -107,7 +107,6 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
@@ -135,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: This plugin allows you to sent events to the Splunk HTTP Event Collector.

data/Gemfile.lock

@@ -1,62 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-splunkhec (1.5)
-      fluentd (>= 0.10.58, < 2)
-      yajl-ruby (>= 1.3.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
-    cool.io (1.5.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    dig_rb (1.0.1)
-    fluentd (1.2.0)
-      cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    hashdiff (0.3.7)
-    http_parser.rb (0.6.0)
-    msgpack (1.2.4)
-    power_assert (1.1.1)
-    public_suffix (3.0.2)
-    rake (0.9.6)
-    safe_yaml (1.0.4)
-    serverengine (2.0.6)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.3)
-    test-unit (3.2.7)
-      power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2018.5)
-      tzinfo (>= 1.0.0)
-    webmock (3.4.1)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff
-    yajl-ruby (1.4.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  fluent-plugin-splunkhec!
-  rake (~> 0.9, >= 0.9.2)
-  test-unit (~> 3.1, >= 3.1.0)
-  webmock (>= 3.0)
-
-BUNDLED WITH
-   1.16.1