fluent-plugin-splunk-hec 1.0.0

data/lib/fluent/plugin/out_splunk_hec/match_formatter.rb

@@ -0,0 +1,22 @@
+require 'fluent/match'
+
+class Fluent::Plugin::SplunkHecOutput::MatchFormatter
+  def initialize(pattern, formatter)
+    # stolen from fluentd/lib/fluent/event_router.rb
+    patterns = pattern.split(/\s+/).map { |str| Fluent::MatchPattern.create(str) }
+    @pattern = if patterns.length == 1
+		 patterns[0]
+	       else
+		 Fluent::OrMatchPattern.new(patterns)
+	       end
+    @formatter = formatter
+  end
+
+  def match?(tag)
+    @pattern.match tag
+  end
+
+  def format(tag, time, record)
+    @formatter.format tag, time, record
+  end
+end

data/lib/fluent/plugin/out_splunk_hec/version.rb

@@ -0,0 +1 @@
+Fluent::Plugin::SplunkHecOutput::VERSION = File.read(File.expand_path('../../../../VERSION', File.dirname(__FILE__))).chomp.strip

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -0,0 +1,271 @@
+require "test_helper"
+
+describe Fluent::Plugin::SplunkHecOutput do
+  include Fluent::Test::Helpers
+  include PluginTestHelper
+
+  before { Fluent::Test.setup } # setup router and others
+    
+  it { expect(::Fluent::Plugin::SplunkHecOutput::VERSION).wont_be_nil }
+
+  describe "hec_host validation" do
+    describe "invalid host" do
+      it "should require hec_host" do
+	expect{ create_output_driver }.must_raise Fluent::ConfigError
+      end
+
+      it { expect{ create_output_driver('hec_host %bad-host%') }.must_raise Fluent::ConfigError }
+    end
+
+    describe "good host" do
+      it {
+	expect(create_output_driver('hec_host splunk.com').instance.hec_host).must_equal "splunk.com"
+      }
+    end
+  end
+
+  it "should send request to Splunk" do
+    req = verify_sent_events { |batch|
+      expect(batch.size).must_equal 2
+    }
+    expect(req).must_be_requested times: 1
+  end
+
+  it "should use host machine's hostname for event host by default" do
+    verify_sent_events { |batch|
+      batch.each do |item|
+	expect(item['host']).must_equal Socket.gethostname
+      end
+    }
+  end
+
+  %w[index source sourcetype].each do |field|
+    it "should not set #{field} by default" do
+      verify_sent_events { |batch|
+	batch.each do |item|
+	  expect(item).wont_include field
+	end
+      }
+    end
+  end
+
+  it "should support ${tag}" do
+    verify_sent_events(<<~CONF) { |batch|
+    index ${tag}
+    host ${tag}
+    source ${tag}
+    sourcetype ${tag}
+    CONF
+      batch.each do |item|
+	%w[index host source sourcetype].each { |field|
+	  expect(%w[tag.event1 tag.event2]).must_include item[field]
+	}
+      end
+    }
+  end
+
+  it "should support *_key" do
+    verify_sent_events(<<~CONF) { |batch|
+      index_key      level
+      host_key       from
+      source_key     file
+      sourcetype_key agent.name
+    CONF
+      batch.each { |item|
+	expect(item['index']).must_equal 'info'
+	expect(item['host']).must_equal 'my_machine'
+	expect(item['source']).must_equal 'cool.log'
+	expect(item['sourcetype']).must_equal 'test'
+
+	JSON.load(item['event']).tap do |event|
+	  %w[level from file].each { |field| expect(event).wont_include field }
+	  expect(event['agent']).wont_include 'name'
+	end
+      }
+    }
+  end
+
+  it "should remove nil fileds." do
+    verify_sent_events(<<~CONF) { |batch|
+      index_key      nonexist
+      host_key       nonexist
+      source_key     nonexist
+      sourcetype_key nonexist
+    CONF
+      batch.each { |item|
+	expect(item).wont_be :has_key?, 'index'
+	expect(item).wont_be :has_key?, 'host'
+	expect(item).wont_be :has_key?, 'source'
+	expect(item).wont_be :has_key?, 'sourcetype'
+      }
+    }
+  end
+
+  describe 'formatter' do
+    it "should support replace the default json formater" do
+      verify_sent_events(<<~CONF) { |batch|
+	<format>
+	  @type single_value
+	  message_key log
+	  add_newline false
+	</format>
+      CONF
+	batch.map { |item| item['event'] }
+	     .each { |event| expect(event).must_equal "everything is good" }
+      }
+    end
+
+    it "should support multiple formatters" do
+      verify_sent_events(<<~CONF) { |batch|
+	source ${tag}
+	<format tag.event1>
+	  @type single_value
+	  message_key log
+	  add_newline false
+	</format>
+      CONF
+	expect(batch.find { |item| item['source'] == 'tag.event1' }['event']).must_equal "everything is good"
+	expect(batch.find { |item| item['source'] == 'tag.event2' }['event']).must_be_instance_of Hash
+      }
+    end
+  end
+
+  it "should support fields for indexed field extraction" do
+    verify_sent_events(<<~CONF) { |batch|
+    <fields>
+      from
+      logLevel level
+      nonexist
+    </fields>
+    CONF
+      batch.each do |item|
+	JSON.load(item['event']).tap { |event|
+	  expect(event).wont_include 'from'
+	  expect(event).wont_include 'level'
+	}
+
+	expect(item['fields']['from']).must_equal 'my_machine'
+	expect(item['fields']['logLevel']).must_equal 'info'
+	expect(item['fields']).wont_be :has_key?, 'nonexist'
+      end
+    }
+  end
+
+  describe 'metric'do
+    it 'should check related configs' do
+      expect(
+	create_output_driver('hec_host somehost', 'data_type metric')
+      ).wont_be_nil
+
+      expect{
+	create_output_driver('hec_host somehost', 'data_type metric', 'metrics_from_event false')
+      }.must_raise Fluent::ConfigError
+
+      expect{
+	create_output_driver('hec_host somehost', 'data_type metric', 'metric_name_key x')
+      }.must_raise Fluent::ConfigError
+
+      expect(
+	create_output_driver('hec_host somehost', 'data_type metric', 'metric_name_key x', 'metric_value_key y')
+      ).wont_be_nil
+    end
+
+    it 'should have "metric" as event, and have proper fields' do
+      verify_sent_events(<<~CONF) { |batch|
+	data_type metric
+	metric_name_key from
+	metric_value_key value
+      CONF
+        batch.each do |item|
+	  expect(item['event']).must_equal 'metric'
+	  expect(item['fields']['metric_name']).must_equal 'my_machine'
+	  expect(item['fields']['_value']).must_equal 100
+	  expect(item['fields']['log']).must_equal 'everything is good'
+	  expect(item['fields']['level']).must_equal 'info'
+	  expect(item['fields']['file']).must_equal 'cool.log'
+	end
+      }
+    end
+
+    it 'should handle empty fields' do
+      verify_sent_events(<<~CONF) { |batch|
+	data_type metric
+	metric_name_key from
+	metric_value_key value
+	<fields>
+	</fields>
+      CONF
+        batch.each do |item|
+	  # only "metric_name" and "_value"
+	  expect(item['fields'].keys.size).must_equal 2
+	end
+      }
+    end
+
+    it 'should handle custom fields' do
+      verify_sent_events(<<~CONF) { |batch|
+	data_type metric
+	metric_name_key from
+	metric_value_key value
+	<fields>
+	  level
+	  filePath file
+	  username
+	</fields>
+      CONF
+        batch.each do |item|
+	  expect(item['fields'].keys.size).must_equal 4
+	  expect(item['fields']['level']).must_equal 'info'
+	  expect(item['fields']['filePath']).must_equal 'cool.log'
+	  # null fields should be removed
+	  expect(item['fields']).wont_be :has_key?, 'username'
+	end
+      }
+    end
+
+    it 'should treat each key-value in event as a metric' do
+      metrics = [
+	['tag', event_time, {'cup': 0.5, 'memory': 100}],
+	['tag', event_time, {'cup': 0.6, 'memory': 200}]
+      ]
+      with_stub_hec(events: metrics, conf: 'data_type metric') { |batch|
+	expect(batch.size).must_equal 4
+      }
+    end
+  end
+
+  def with_stub_hec(events:, conf: '', &blk)
+    host = "hec.splunk.com"
+    @driver = create_output_driver("hec_host #{host}", conf)
+
+    hec_req = stub_hec_request("https://#{host}:8088").with { |r|
+      blk.call r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }
+    }
+
+    @driver.run do
+      events.each { |evt| @driver.feed *evt }
+    end
+
+    hec_req
+  end
+
+  def verify_sent_events(conf = '', &blk)
+    event = {
+      "log"   => "everything is good",
+      "level" => "info",
+      "from"  => "my_machine",
+      "file"  => "cool.log",
+      "value" => 100,
+      "agent" => {
+	"name"    => "test",
+	"version" => "1.0.0"
+      }
+    }
+    events = [
+      ["tag.event1", event_time, {"id" => "1st"}.merge(Marshal.load(Marshal.dump(event)))],
+      ["tag.event2", event_time, {"id" => "2nd"}.merge(Marshal.load(Marshal.dump(event)))]
+    ]
+
+    with_stub_hec conf: conf, events: events, &blk
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,39 @@
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "fluent/plugin/out_splunk_hec"
+
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+require "minitest/autorun"
+require "webmock/minitest"
+
+
+# make assertions from webmock available in minitest/spec
+module Minitest::Expectations
+  infect_an_assertion :assert_requested, :must_be_requested, :reverse
+  infect_an_assertion :assert_not_requested, :wont_be_requested, :reverse
+end
+
+TEST_HEC_TOKEN = "some-token".freeze
+
+module PluginTestHelper
+  def fluentd_conf_for(*lines)
+    basic_config = [
+      "hec_token #{TEST_HEC_TOKEN}"
+    ]
+    (basic_config + lines).join("\n")
+  end
+
+  def create_output_driver(*configs)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHecOutput).tap { |d|
+      d.configure(fluentd_conf_for(*configs))
+    }
+  end
+
+  def stub_hec_request(endpoint)
+    stub_request(:post, "#{endpoint}/services/collector").
+      with(headers: {"Authorization" => "Splunk #{TEST_HEC_TOKEN}", "User-Agent" => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}"}).
+      to_return(body: '{"text":"Success","code":0}')
+  end
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-splunk-hec
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Zhimin (Gimi) Liang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: A fluentd output plugin created by Splunk that writes events to splunk
+  indexers over HTTP Event Collector API.
+email:
+- zliang@splunk.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-splunk-hec.gemspec
+- lib/fluent/plugin/out_splunk_hec.rb
+- lib/fluent/plugin/out_splunk_hec/match_formatter.rb
+- lib/fluent/plugin/out_splunk_hec/version.rb
+- test/fluent/plugin/out_splunk_hec_test.rb
+- test/lib/webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/webmock/http_lib_adapters/excon_adapter.rb
+- test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/httpclient_adapter.rb
+- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/test_helper.rb
+homepage: https://github.com/splunk/fluent-plugin-splunk-hec
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin for Splunk HEC.
+test_files:
+- test/fluent/plugin/out_splunk_hec_test.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
+- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/webmock/http_lib_adapters/httpclient_adapter.rb
+- test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/excon_adapter.rb
+- test/test_helper.rb