fluent-plugin-splunk-hec 1.2.13 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de7c88e53248a080bb6074bf72a88896438727d88a042945b1faacce56df8a34
-  data.tar.gz: 44a74cb51c4697397ef0fac7d50e6ff9a044e823c336c5a680840e387b8383e5
+  metadata.gz: 6a8a63dd4210f66f194a029bceed3d3c407aaa3fb764b69961fc01efcb801d9e
+  data.tar.gz: 8aab1ee69dd9c4cdcc2422da2720db89987c270b0acf4bcd01b97a908cf14763
 SHA512:
-  metadata.gz: fa2ef5e67fbb22bc0721e15cb9d73e9442f5485dc70da665507455e9cba99ec8c672ea65c308b50fd82249977ee9f06a3120c6d5be56b8ef8251e6927f4e1d29
-  data.tar.gz: 9945827eb5330b28eaa319c7de17825bb1f6c5ea9494c5bdeca128eb9839663318f698d9f5ba7718c62c50c0095a3ca9456951293a93d9aa00dc8de10edfb66c
+  metadata.gz: 4fe2f5eb7e22e8e9914754507add8d06df80be96b8fe8d2d57ce295730f77d948c16dde1bda4199ac63f2ab8798027841ec694c99113558405ad49d9000b50ab
+  data.tar.gz: f2e69d508a72eb1f1e306e66f1defa6572e636f148e314c9cd26fee1d7f3c2b5dd437e65c41c5e7e6ee2a77e0f0aeb341d5d849ab60313ae3fefba096ce8a1d5

data/Gemfile

@@ -4,6 +4,7 @@ source 'https://rubygems.org'
 
 group :test do
   gem 'simplecov', require: false
+  gem 'net-smtp', require: false
 end
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-splunk-hec (1.2.13)
-      fluentd (>= 1.4)
+    fluent-plugin-splunk-hec (1.3.1)
+      fluentd (>= 1.5)
+      json-jwt (~> 1.15.0)
       multi_json (~> 1.13)
       net-http-persistent (~> 4.0)
       openid_connect (~> 1.1.8)
       prometheus-client (>= 2.1.0)
+      rack-oauth2 (~> 1.19)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activesupport (7.0.2.3)
+    activemodel (7.0.4)
+      activesupport (= 7.0.4)
+    activesupport (7.0.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -22,19 +24,26 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     aes_key_wrap (1.1.0)
     attr_required (1.0.1)
-    bindata (2.4.10)
+    bindata (2.4.14)
     concurrent-ruby (1.1.10)
-    connection_pool (2.2.5)
+    connection_pool (2.3.0)
     cool.io (1.7.1)
     crack (0.4.5)
       rexml
+    digest (3.1.0)
     docile (1.4.0)
-    fluentd (1.14.6)
+    faraday (2.7.1)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-follow_redirects (0.3.0)
+      faraday (>= 1, < 3)
+    faraday-net_http (3.0.2)
+    fluentd (1.15.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -44,20 +53,29 @@ GEM
     hashdiff (1.0.1)
     http_parser.rb (0.8.0)
     httpclient (2.8.3)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    json-jwt (1.13.0)
+    io-wait (0.2.1)
+    json-jwt (1.15.3)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
+      httpclient
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     mini_mime (1.1.2)
     minitest (5.15.0)
-    msgpack (1.5.1)
+    msgpack (1.6.0)
     multi_json (1.15.0)
     net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
+    net-protocol (0.1.2)
+      io-wait
+      timeout
+    net-smtp (0.3.1)
+      digest
+      net-protocol
+      timeout
     openid_connect (1.1.8)
       activemodel
       attr_required (>= 1.0.0)
@@ -71,8 +89,8 @@ GEM
     power_assert (2.0.1)
     prometheus-client (4.0.0)
     public_suffix (4.0.6)
-    rack (2.2.3)
-    rack-oauth2 (1.19.0)
+    rack (3.0.1)
+    rack-oauth2 (1.21.2)
       activesupport
       attr_required
       httpclient
@@ -80,7 +98,8 @@ GEM
       rack (>= 2.1.0)
     rake (13.0.6)
     rexml (3.2.5)
-    serverengine (2.2.5)
+    ruby2_keywords (0.0.5)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -90,31 +109,34 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.3)
     strptime (0.2.5)
-    swd (1.3.0)
+    swd (2.0.2)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
-      httpclient (>= 2.4)
+      faraday (~> 2.0)
+      faraday-follow_redirects
     test-unit (3.5.3)
       power_assert
-    tzinfo (2.0.4)
+    timeout (0.2.0)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2022.1)
+    tzinfo-data (1.2022.6)
       tzinfo (>= 1.0.0)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
-    validate_url (1.0.13)
+    validate_url (1.0.15)
       activemodel (>= 3.0.0)
       public_suffix
-    webfinger (1.2.0)
+    webfinger (2.1.2)
       activesupport
-      httpclient (>= 2.4)
+      faraday (~> 2.0)
+      faraday-follow_redirects
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
     webrick (1.7.0)
-    yajl-ruby (1.4.2)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -123,10 +145,11 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-splunk-hec!
   minitest (~> 5.0)
+  net-smtp
   rake (>= 12.0)
   simplecov
   test-unit (~> 3.0)
   webmock (~> 3.5.0)
 
 BUNDLED WITH
-   2.3.12
+   2.3.26

data/README.md

@@ -400,6 +400,9 @@ Specifies which formatter to use.
 
 The following parameters can be used for tuning HTTP connections:
 
+#### gzip_compression (boolean)
+Whether to use gzip compression on outbound posts. This parameter is set to `false` by default for backwards compatibility.
+
 #### idle_timeout (integer)
 
 The default is five seconds. If a connection has not been used for five seconds, it is automatically reset at next use, in order to avoid attempting to send to a closed connection. Specifiy `nil` to prohibit any timeouts.

data/VERSION

@@ -1 +1 @@
-1.2.13
+1.3.1

data/fluent-plugin-splunk-hec.gemspec

@@ -33,11 +33,14 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_runtime_dependency 'fluentd', '>= 1.4'
+  spec.add_runtime_dependency 'fluentd', '>= 1.5'
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
   spec.add_runtime_dependency 'net-http-persistent', '~> 4.0'
   spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'
   spec.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
+  spec.add_runtime_dependency 'json-jwt', '~> 1.15.0'
+  spec.add_runtime_dependency 'rack-oauth2', '~> 1.19'
+
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '>= 12.0'

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -9,6 +9,7 @@ require 'fluent/plugin/out_splunk'
 require 'openssl'
 require 'multi_json'
 require 'net/http/persistent'
+require 'zlib'
 
 module Fluent::Plugin
   class SplunkHecOutput < SplunkOutput
@@ -96,6 +97,9 @@ module Fluent::Plugin
     desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
     config_param :keep_keys, :bool, default: false
 
+    desc 'Indicates if GZIP Compression is enabled.'
+    config_param :gzip_compression, :bool, default: false
+
     desc 'App name'
     config_param :app_name, :string, default: "hec_plugin_gem"
 
@@ -321,13 +325,20 @@ module Fluent::Plugin
         c.override_headers['Authorization'] = "Splunk #{@hec_token}"
         c.override_headers['__splunk_app_name'] = "#{@app_name}"
         c.override_headers['__splunk_app_version'] = "#{@app_version}"
-
       end
     end
 
     def write_to_splunk(chunk)
       post = Net::HTTP::Post.new @api.request_uri
-      post.body = chunk.read
+      if @gzip_compression
+        post.add_field("Content-Encoding", "gzip")
+        gzip_stream = Zlib::GzipWriter.new StringIO.new
+        gzip_stream << chunk.read
+        post.body = gzip_stream.close.string
+      else
+        post.body = chunk.read
+      end
+
       log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
       log.trace { "POST #{@api} body=#{post.body}" }
       begin

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -60,6 +60,12 @@ describe Fluent::Plugin::SplunkHecOutput do
     it 'should consume chunks on 4xx errors' do
       expect(create_hec_output_driver('hec_host hec_token').instance.consume_chunk_on_4xx_errors).must_equal true
     end
+    it 'should default gzip off' do
+      expect(create_hec_output_driver('hec_host hec_token').instance.gzip_compression).must_equal false
+    end
+    it 'should support enabling gzip' do
+      expect(create_hec_output_driver('hec_host hec_token', 'gzip_compression true').instance.gzip_compression).must_equal true
+    end
   end
 
   describe 'hec_host validation' do
@@ -357,6 +363,15 @@ describe Fluent::Plugin::SplunkHecOutput do
     end
   end
 
+  describe 'gzip encoding' do
+    it 'should include gzip header when enabled' do
+      metrics = [
+        ['tag', event_time, { 'cup': 0.5, 'memory': 100 }]
+      ]
+      with_stub_hec_gzip(events: metrics, conf: 'data_type metric')
+    end
+  end
+
   def with_stub_hec(events:, conf: '')
     host = 'hec.splunk.com'
     @driver = create_hec_output_driver("hec_host #{host}", conf)
@@ -372,6 +387,21 @@ describe Fluent::Plugin::SplunkHecOutput do
     hec_req
   end
 
+  def with_stub_hec_gzip(events:, conf: '')
+    host = 'hec.splunk.com'
+    @driver = create_hec_output_driver("hec_host #{host}", 'gzip_compression true', conf)
+
+    hec_req = stub_hec_gzip_request("https://#{host}:8088").with do |r|
+      yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }
+    end
+
+    @driver.run do
+      events.each { |evt| @driver.feed *evt }
+    end
+
+    hec_req
+  end
+
   def verify_sent_events(conf = '', &blk)
     event = {
       'log' => 'everything is good',

data/test/test_helper.rb

@@ -42,4 +42,15 @@ module PluginTestHelper
                        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" })
       .to_return(body: '{"text":"Success","code":0}')
   end
+
+  def stub_hec_gzip_request(endpoint)
+    stub_request(:post, "#{endpoint}/services/collector")
+      .with(headers: {
+        'Authorization' => "Splunk #{TEST_HEC_TOKEN}",
+        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}",
+        'Content-Encoding' => "gzip"
+      },
+      )
+      .to_return(body: '{"text":"GzipSuccess","code":0}')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.2.13
+  version: 1.3.1
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-29 00:00:00.000000000 Z
+date: 2022-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: json-jwt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+- !ruby/object:Gem::Dependency
+  name: rack-oauth2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -221,13 +249,13 @@ signing_key:
 specification_version: 4
 summary: Fluentd plugin for Splunk HEC.
 test_files:
-- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
+- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/webmock/http_lib_adapters/curb_adapter.rb
 - test/lib/webmock/http_lib_adapters/patron_adapter.rb
-- test/lib/webmock/http_lib_adapters/excon_adapter.rb
 - test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
-- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/webmock/http_lib_adapters/excon_adapter.rb
 - test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
-- test/lib/webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/test_helper.rb
-- test/fluent/plugin/out_splunk_hec_test.rb
 - test/fluent/plugin/out_splunk_ingest_api_test.rb
+- test/fluent/plugin/out_splunk_hec_test.rb