fluent-plugin-splunk-hec 1.2.13 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de7c88e53248a080bb6074bf72a88896438727d88a042945b1faacce56df8a34
-  data.tar.gz: 44a74cb51c4697397ef0fac7d50e6ff9a044e823c336c5a680840e387b8383e5
+  metadata.gz: 23746f631e5a9217fca76160c200155878aecdc2402dcaa0dec2b7447f7f5986
+  data.tar.gz: 43e704a9048113bbc9070972a809ab2d689e05ba63c2a1739d35abad1ea7ba08
 SHA512:
-  metadata.gz: fa2ef5e67fbb22bc0721e15cb9d73e9442f5485dc70da665507455e9cba99ec8c672ea65c308b50fd82249977ee9f06a3120c6d5be56b8ef8251e6927f4e1d29
-  data.tar.gz: 9945827eb5330b28eaa319c7de17825bb1f6c5ea9494c5bdeca128eb9839663318f698d9f5ba7718c62c50c0095a3ca9456951293a93d9aa00dc8de10edfb66c
+  metadata.gz: 531cb324fcfdbdd81b89abca06d39ec324776dd973c23118eda9df697160980f87f43dd1beded8adec71103f9ed1bcf43ced51b60b30f6ebce032516524c3424
+  data.tar.gz: 27cdfef7e22314183999e93651275da29ca4c473b44d68c8873c458f9fcbf1d7213b1632690d4dd7b923699822cf7b57fd4fdea0b6a48b7dc51767d9785a1176

data/Gemfile

@@ -4,6 +4,7 @@ source 'https://rubygems.org'
 
 group :test do
   gem 'simplecov', require: false
+  gem 'net-smtp', require: false
 end
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-splunk-hec (1.2.13)
-      fluentd (>= 1.4)
+    fluent-plugin-splunk-hec (1.3.0)
+      fluentd (>= 1.5)
       multi_json (~> 1.13)
       net-http-persistent (~> 4.0)
       openid_connect (~> 1.1.8)
@@ -11,9 +11,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activesupport (7.0.2.3)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,13 +28,14 @@ GEM
     cool.io (1.7.1)
     crack (0.4.5)
       rexml
+    digest (3.1.0)
     docile (1.4.0)
-    fluentd (1.14.6)
+    fluentd (1.15.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -46,6 +47,7 @@ GEM
     httpclient (2.8.3)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    io-wait (0.2.1)
     json-jwt (1.13.0)
       activesupport (>= 4.2)
       aes_key_wrap
@@ -54,10 +56,17 @@ GEM
       mini_mime (>= 0.1.1)
     mini_mime (1.1.2)
     minitest (5.15.0)
-    msgpack (1.5.1)
+    msgpack (1.5.4)
     multi_json (1.15.0)
     net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
+    net-protocol (0.1.2)
+      io-wait
+      timeout
+    net-smtp (0.3.1)
+      digest
+      net-protocol
+      timeout
     openid_connect (1.1.8)
       activemodel
       attr_required (>= 1.0.0)
@@ -71,7 +80,7 @@ GEM
     power_assert (2.0.1)
     prometheus-client (4.0.0)
     public_suffix (4.0.6)
-    rack (2.2.3)
+    rack (2.2.3.1)
     rack-oauth2 (1.19.0)
       activesupport
       attr_required
@@ -80,7 +89,7 @@ GEM
       rack (>= 2.1.0)
     rake (13.0.6)
     rexml (3.2.5)
-    serverengine (2.2.5)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -96,14 +105,15 @@ GEM
       httpclient (>= 2.4)
     test-unit (3.5.3)
       power_assert
-    tzinfo (2.0.4)
+    timeout (0.2.0)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2022.1)
+    tzinfo-data (1.2022.2)
       tzinfo (>= 1.0.0)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
-    validate_url (1.0.13)
+    validate_url (1.0.15)
       activemodel (>= 3.0.0)
       public_suffix
     webfinger (1.2.0)
@@ -114,7 +124,7 @@ GEM
       crack (>= 0.3.2)
       hashdiff
     webrick (1.7.0)
-    yajl-ruby (1.4.2)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -123,10 +133,11 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-splunk-hec!
   minitest (~> 5.0)
+  net-smtp
   rake (>= 12.0)
   simplecov
   test-unit (~> 3.0)
   webmock (~> 3.5.0)
 
 BUNDLED WITH
-   2.3.12
+   2.3.20

data/VERSION

@@ -1 +1 @@
-1.2.13
+1.3.0

data/fluent-plugin-splunk-hec.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_runtime_dependency 'fluentd', '>= 1.4'
+  spec.add_runtime_dependency 'fluentd', '>= 1.5'
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
   spec.add_runtime_dependency 'net-http-persistent', '~> 4.0'
   spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -9,6 +9,7 @@ require 'fluent/plugin/out_splunk'
 require 'openssl'
 require 'multi_json'
 require 'net/http/persistent'
+require 'zlib'
 
 module Fluent::Plugin
   class SplunkHecOutput < SplunkOutput
@@ -96,6 +97,9 @@ module Fluent::Plugin
     desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
     config_param :keep_keys, :bool, default: false
 
+    desc 'Indicates if GZIP Compression is enabled.'
+    config_param :gzip_compression, :bool, default: false
+
     desc 'App name'
     config_param :app_name, :string, default: "hec_plugin_gem"
 
@@ -321,13 +325,20 @@ module Fluent::Plugin
         c.override_headers['Authorization'] = "Splunk #{@hec_token}"
         c.override_headers['__splunk_app_name'] = "#{@app_name}"
         c.override_headers['__splunk_app_version'] = "#{@app_version}"
-
       end
     end
 
     def write_to_splunk(chunk)
       post = Net::HTTP::Post.new @api.request_uri
-      post.body = chunk.read
+      if @gzip_compression
+        post.add_field("Content-Encoding", "gzip")
+        gzip_stream = Zlib::GzipWriter.new StringIO.new
+        gzip_stream << chunk.read
+        post.body = gzip_stream.close.string
+      else
+        post.body = chunk.read
+      end
+
       log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
       log.trace { "POST #{@api} body=#{post.body}" }
       begin

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -60,6 +60,12 @@ describe Fluent::Plugin::SplunkHecOutput do
     it 'should consume chunks on 4xx errors' do
       expect(create_hec_output_driver('hec_host hec_token').instance.consume_chunk_on_4xx_errors).must_equal true
     end
+    it 'should default gzip off' do
+      expect(create_hec_output_driver('hec_host hec_token').instance.gzip_compression).must_equal false
+    end
+    it 'should support enabling gzip' do
+      expect(create_hec_output_driver('hec_host hec_token', 'gzip_compression true').instance.gzip_compression).must_equal true
+    end
   end
 
   describe 'hec_host validation' do
@@ -357,6 +363,15 @@ describe Fluent::Plugin::SplunkHecOutput do
     end
   end
 
+  describe 'gzip encoding' do
+    it 'should include gzip header when enabled' do
+      metrics = [
+        ['tag', event_time, { 'cup': 0.5, 'memory': 100 }]
+      ]
+      with_stub_hec_gzip(events: metrics, conf: 'data_type metric')
+    end
+  end
+
   def with_stub_hec(events:, conf: '')
     host = 'hec.splunk.com'
     @driver = create_hec_output_driver("hec_host #{host}", conf)
@@ -372,6 +387,21 @@ describe Fluent::Plugin::SplunkHecOutput do
     hec_req
   end
 
+  def with_stub_hec_gzip(events:, conf: '')
+    host = 'hec.splunk.com'
+    @driver = create_hec_output_driver("hec_host #{host}", 'gzip_compression true', conf)
+
+    hec_req = stub_hec_gzip_request("https://#{host}:8088").with do |r|
+      yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }
+    end
+
+    @driver.run do
+      events.each { |evt| @driver.feed *evt }
+    end
+
+    hec_req
+  end
+
   def verify_sent_events(conf = '', &blk)
     event = {
       'log' => 'everything is good',

data/test/test_helper.rb

@@ -42,4 +42,15 @@ module PluginTestHelper
                        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" })
       .to_return(body: '{"text":"Success","code":0}')
   end
+
+  def stub_hec_gzip_request(endpoint)
+    stub_request(:post, "#{endpoint}/services/collector")
+      .with(headers: {
+        'Authorization' => "Splunk #{TEST_HEC_TOKEN}",
+        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}",
+        'Content-Encoding' => "gzip"
+      },
+      )
+      .to_return(body: '{"text":"GzipSuccess","code":0}')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.2.13
+  version: 1.3.0
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-29 00:00:00.000000000 Z
+date: 2022-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -221,13 +221,13 @@ signing_key:
 specification_version: 4
 summary: Fluentd plugin for Splunk HEC.
 test_files:
-- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
-- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/test_helper.rb
 - test/lib/webmock/http_lib_adapters/excon_adapter.rb
-- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
 - test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
-- test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/lib/webmock/http_lib_adapters/curb_adapter.rb
-- test/test_helper.rb
+- test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
 - test/fluent/plugin/out_splunk_hec_test.rb
 - test/fluent/plugin/out_splunk_ingest_api_test.rb