fluent-plugin-splunk-hec 1.1.2 → 1.2.0

data/test/fluent/plugin/out_splunk_ingest_api_test.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+describe Fluent::Plugin::SplunkIngestApiOutput do
+  include Fluent::Test::Helpers
+  include PluginTestHelper
+
+  INGEST_API_ENDPOINT = 'https://api.splunkbeta.com/tenant_name/ingest/v1beta2/events'
+  AUTH_TOKEN_ENDPOINT = 'https://api.splunkbeta.com/system/identity/v1/token'
+
+  before { Fluent::Test.setup } # setup router and others
+
+  it { expect(::Fluent::Plugin::SplunkIngestApiOutput::VERSION).wont_be_nil }
+
+  describe 'Required configs validation' do
+    it 'should have required fields' do
+      expect { create_api_output_driver }.must_raise Fluent::ConfigError
+    end
+
+    describe 'good_config' do
+      it {
+        instance = create_api_output_driver('service_client_identifier service_client_id',
+                                            'service_client_secret_key secret_key',
+                                            'ingest_api_tenant tenant_name').instance
+        expect(instance.service_client_identifier).must_equal 'service_client_id'
+        expect(instance.service_client_secret_key).must_equal 'secret_key'
+        expect(instance.ingest_api_tenant).must_equal 'tenant_name'
+      }
+    end
+    describe 'invalid host' do
+      it {
+        expect do
+          create_api_output_driver('ingest_api_host %bad-host%',
+                                   'service_client_identifier service_client_id',
+                                   'service_client_secret_key secret_key',
+                                   'ingest_api_tenant tenant_name')
+        end.must_raise Fluent::ConfigError
+      }
+    end
+    describe 'missing tenant name' do
+      it {
+        expect do
+          create_api_output_driver('ingest_api_host %bad-host%',
+                                   'service_client_identifier service_client_id',
+                                   'service_client_secret_key secret_key',
+                                   'ingest_api_tenant tenant_name')
+        end.must_raise Fluent::ConfigError
+      }
+    end
+    describe 'missing client identifier' do
+      it {
+        expect do
+          create_api_output_driver('ingest_api_host %bad-host%',
+                                   'service_client_secret_key secret_key',
+                                   'ingest_api_tenant tenant_name')
+        end.must_raise Fluent::ConfigError
+      }
+    end
+
+    describe 'missing secret key' do
+      it {
+        expect do
+          create_api_output_driver('ingest_api_host %bad-host%',
+                                   'service_client_identifier service_client_id',
+                                   'ingest_api_tenant tenant_name')
+        end.must_raise Fluent::ConfigError
+      }
+    end
+  end
+
+  it 'should not fail to start when provided bad auth' do
+    stub_failed_auth
+    driver = create_api_output_driver('service_client_identifier service_client_id',
+                                      'service_client_secret_key secret_key',
+                                      'ingest_api_tenant tenant_name')
+    driver.run
+  end
+
+  it 'should send request to Splunk' do
+    req = verify_sent_events do |batch|
+      expect(batch.size).must_equal 2
+    end
+    expect(req).must_be_requested times: 1
+  end
+
+  it 'should have an index in the attributes slot' do
+    verify_sent_events(conf: %(
+      index my_index
+    )) do |batch|
+      batch.each do |item|
+        expect(item['attributes']['index']).must_equal 'my_index'
+      end
+    end
+  end
+
+  it 'should have attrbutes not fields' do
+    verify_sent_events do |batch|
+      batch.each do |item|
+        expect(item).wont_include :fields
+        expect(item).includes :attributes
+      end
+    end
+  end
+
+  it 'should have body not event' do
+    verify_sent_events do |batch|
+      batch.each do |item|
+        expect(item).wont_include :event
+        expect(item).includes :body
+      end
+    end
+  end
+
+  it 'should have a timestamp and nanos' do
+    verify_sent_events do |batch|
+      batch.each do |item|
+        expect(item).wont_include :time
+        expect(item).includes :timestamp
+        expect(item).includes :nanos
+      end
+    end
+  end
+
+  it 'should raise error on 401/429 to force retry' do
+    # Try to quiet this down some.
+    report_on_exception = Thread.report_on_exception
+    Thread.report_on_exception = false
+    begin
+      expect do
+        verify_sent_events status: 429
+      end.must_raise RuntimeError
+
+      expect do
+        verify_sent_events status: 401
+      end.must_raise RuntimeError
+    ensure
+      Thread.report_on_exception = report_on_exception
+    end
+  end
+
+  it 'should not send an empty log message' do
+    verify_sent_events conf: %(
+      <format>
+        @type single_value
+        message_key log
+        add_newline false
+     </format>), event: { 'log' => "\n" } do |batch|
+      batch.each do |_item|
+        raise 'No message should be sent'
+      end
+    end
+  end
+
+  # it 'should send index from filters' do
+  #   verify_sent_events conf: %[
+  #     <filter>
+  #       @type record_transformer
+  #       enable_ruby
+  #       <record>
+  #           index ${ENV['SPLUNK_INDEX']}
+  #       </record>
+  #     </filter>
+  #   ], event: {"log" => "This is the log", "index" => "indexname"} do |batch|
+  #     batch.each do |item|
+  #       item[:attrbutes][:index].
+  #       fail "No message should be sent"
+  #     end
+  #   end
+  # end
+
+  def create_api_output_driver(*configs)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkIngestApiOutput).tap do |d|
+      d.configure(configs.join("\n"))
+    end
+  end
+
+  DEFAULT_EVENT = {
+    log: 'everything is good',
+    level: 'info',
+    from: 'my_machine',
+    file: 'cool.log',
+    value: 100,
+    agent: {
+      name: 'test',
+      version: '1.0.0'
+    }
+  }.freeze
+
+  def verify_sent_events(args = {})
+    conf = args[:conf] || ''
+    event = args[:event] || DEFAULT_EVENT
+    status = args[:status] || 200
+
+    events = [
+      ['tag.event1', event_time, { id: '1st' }.merge(Marshal.load(Marshal.dump(event)))],
+      ['tag.event2', event_time, { id: '2nd' }.merge(Marshal.load(Marshal.dump(event)))]
+    ]
+
+    @driver = create_api_output_driver('service_client_identifier service_client_id',
+                                       'service_client_secret_key secret_key',
+                                       'ingest_api_tenant tenant_name',
+                                       conf)
+
+    api_req = if status == 200
+                stub_successful_api_request.with do |r|
+                  yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }.first
+                end
+              else
+                stub_failed_api_request status
+              end
+
+    @driver.run do
+      events.each { |evt| @driver.feed *evt }
+    end
+
+    api_req
+  end
+
+  def stub_successful_auth
+    stub_request(:post, AUTH_TOKEN_ENDPOINT)
+      .to_return(body: '{"access_token":"bearer token","token_type":"Bearer","expires_in":432000,"scope":"client_credentials"}')
+  end
+
+  def stub_failed_auth
+    stub_request(:post, AUTH_TOKEN_ENDPOINT)
+      .to_return(status: 401,
+                 body: '{"error":"invalid_client","error_description":"The client secret supplied for a confidential client is invalid."}')
+  end
+
+  def stub_successful_api_request
+    stub_successful_auth
+
+    stub_request(:post, INGEST_API_ENDPOINT)
+      .to_return(body: '{"message":"Success","code":"SUCCESS"}')
+  end
+
+  def stub_failed_api_request(status)
+    stub_successful_auth
+
+    stub_request(:post, INGEST_API_ENDPOINT)
+      .to_return(body: '', status: status)
+  end
+end

data/test/test_helper.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 require 'simplecov'
 SimpleCov.start
 
-$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift File.expand_path('lib', __dir__)
 require 'fluent/plugin/out_splunk_hec'
+require 'fluent/plugin/out_splunk_ingest_api'
 
 require 'fluent/test'
 require 'fluent/test/driver/output'
@@ -17,7 +20,7 @@ module Minitest::Expectations
   infect_an_assertion :assert_not_requested, :wont_be_requested, :reverse
 end
 
-TEST_HEC_TOKEN = 'some-token'.freeze
+TEST_HEC_TOKEN = 'some-token'
 
 module PluginTestHelper
   def fluentd_conf_for(*lines)
@@ -27,16 +30,16 @@ module PluginTestHelper
     (basic_config + lines).join("\n")
   end
 
-  def create_output_driver(*configs)
-    Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHecOutput).tap { |d|
+  def create_hec_output_driver(*configs)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHecOutput).tap do |d|
       d.configure(fluentd_conf_for(*configs))
-    }
+    end
   end
 
   def stub_hec_request(endpoint)
     stub_request(:post, "#{endpoint}/services/collector")
       .with(headers: { 'Authorization' => "Splunk #{TEST_HEC_TOKEN}",
-                         'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" })
+                       'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" })
       .to_return(body: '{"text":"Success","code":0}')
   end
 end

metadata

@@ -1,27 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-12 00:00:00.000000000 Z
+date: 2019-10-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-kubernetes_metadata_filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.2
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: '1.4'
 - !ruby/object:Gem::Dependency
@@ -52,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: openid_connect
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.6
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -81,61 +123,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '12.0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.63.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.63.1
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 0.16.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 0.16.1
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.1
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.1
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
 description: |-
   A fluentd output plugin created by Splunk
     that writes events to splunk indexers over HTTP Event Collector API.
@@ -153,15 +209,18 @@ files:
 - Rakefile
 - VERSION
 - fluent-plugin-splunk-hec.gemspec
+- lib/fluent/plugin/out_splunk.rb
+- lib/fluent/plugin/out_splunk/match_formatter.rb
+- lib/fluent/plugin/out_splunk/version.rb
 - lib/fluent/plugin/out_splunk_hec.rb
-- lib/fluent/plugin/out_splunk_hec/match_formatter.rb
 - lib/fluent/plugin/out_splunk_hec/version.rb
+- lib/fluent/plugin/out_splunk_ingest_api.rb
 - test/fluent/plugin/out_splunk_hec_test.rb
+- test/fluent/plugin/out_splunk_ingest_api_test.rb
 - test/lib/webmock/http_lib_adapters/curb_adapter.rb
 - test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
 - test/lib/webmock/http_lib_adapters/excon_adapter.rb
 - test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
-- test/lib/webmock/http_lib_adapters/httpclient_adapter.rb
 - test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/lib/webmock/http_lib_adapters/patron_adapter.rb
 - test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
@@ -185,18 +244,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Splunk HEC.
 test_files:
-- test/fluent/plugin/out_splunk_hec_test.rb
-- test/lib/webmock/http_lib_adapters/patron_adapter.rb
-- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
 - test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
 - test/lib/webmock/http_lib_adapters/curb_adapter.rb
-- test/lib/webmock/http_lib_adapters/httpclient_adapter.rb
+- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
 - test/lib/webmock/http_lib_adapters/excon_adapter.rb
+- test/fluent/plugin/out_splunk_ingest_api_test.rb
+- test/fluent/plugin/out_splunk_hec_test.rb
 - test/test_helper.rb