fluent-plugin-splunk-hec 1.2.11 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 277de2e282c7dfee68431d0b4ca144b8215d68f7210814fff1caadc452527955
-  data.tar.gz: 99bd20f106656d0be1ab6d70c3d1afd4516fbd19995e3edc8c6357f123f18e12
+  metadata.gz: 23746f631e5a9217fca76160c200155878aecdc2402dcaa0dec2b7447f7f5986
+  data.tar.gz: 43e704a9048113bbc9070972a809ab2d689e05ba63c2a1739d35abad1ea7ba08
 SHA512:
-  metadata.gz: e8b91a7bfc42ce39100754bd79730a7fad4b27337e9cd70e13ccbc836f05aa98e123714cf268cd8f76bb351b8e409191050d8a9b4856bc088334a7a10f174000
-  data.tar.gz: 8874436fb7b143529994256d7f8d99d0f57c113e40ff1b3907af0a08c802a65372b7550c70d7db35af75c4de9e826385d0f7a663f82f8941062a2f302c46087a
+  metadata.gz: 531cb324fcfdbdd81b89abca06d39ec324776dd973c23118eda9df697160980f87f43dd1beded8adec71103f9ed1bcf43ced51b60b30f6ebce032516524c3424
+  data.tar.gz: 27cdfef7e22314183999e93651275da29ca4c473b44d68c8873c458f9fcbf1d7213b1632690d4dd7b923699822cf7b57fd4fdea0b6a48b7dc51767d9785a1176

data/Gemfile

@@ -4,6 +4,7 @@ source 'https://rubygems.org'
 
 group :test do
   gem 'simplecov', require: false
+  gem 'net-smtp', require: false
 end
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-splunk-hec (1.2.11)
-      fluentd (>= 1.4)
+    fluent-plugin-splunk-hec (1.3.0)
+      fluentd (>= 1.5)
       multi_json (~> 1.13)
-      net-http-persistent (~> 3.1)
+      net-http-persistent (~> 4.0)
       openid_connect (~> 1.1.8)
       prometheus-client (>= 2.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.1)
-      activesupport (= 7.0.1)
-    activesupport (7.0.1)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -23,18 +23,19 @@ GEM
     aes_key_wrap (1.1.0)
     attr_required (1.0.1)
     bindata (2.4.10)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     connection_pool (2.2.5)
     cool.io (1.7.1)
     crack (0.4.5)
       rexml
+    digest (3.1.0)
     docile (1.4.0)
-    fluentd (1.14.4)
+    fluentd (1.15.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -44,8 +45,9 @@ GEM
     hashdiff (1.0.1)
     http_parser.rb (0.8.0)
     httpclient (2.8.3)
-    i18n (1.9.1)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    io-wait (0.2.1)
     json-jwt (1.13.0)
       activesupport (>= 4.2)
       aes_key_wrap
@@ -54,10 +56,17 @@ GEM
       mini_mime (>= 0.1.1)
     mini_mime (1.1.2)
     minitest (5.15.0)
-    msgpack (1.4.4)
+    msgpack (1.5.4)
     multi_json (1.15.0)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
+    net-protocol (0.1.2)
+      io-wait
+      timeout
+    net-smtp (0.3.1)
+      digest
+      net-protocol
+      timeout
     openid_connect (1.1.8)
       activemodel
       attr_required (>= 1.0.0)
@@ -69,9 +78,9 @@ GEM
       validate_url
       webfinger (>= 1.0.1)
     power_assert (2.0.1)
-    prometheus-client (2.1.0)
+    prometheus-client (4.0.0)
     public_suffix (4.0.6)
-    rack (2.2.3)
+    rack (2.2.3.1)
     rack-oauth2 (1.19.0)
       activesupport
       attr_required
@@ -80,7 +89,7 @@ GEM
       rack (>= 2.1.0)
     rake (13.0.6)
     rexml (3.2.5)
-    serverengine (2.2.5)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -96,14 +105,15 @@ GEM
       httpclient (>= 2.4)
     test-unit (3.5.3)
       power_assert
-    tzinfo (2.0.4)
+    timeout (0.2.0)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.5)
+    tzinfo-data (1.2022.2)
       tzinfo (>= 1.0.0)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
-    validate_url (1.0.13)
+    validate_url (1.0.15)
       activemodel (>= 3.0.0)
       public_suffix
     webfinger (1.2.0)
@@ -114,7 +124,7 @@ GEM
       crack (>= 0.3.2)
       hashdiff
     webrick (1.7.0)
-    yajl-ruby (1.4.1)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -123,10 +133,11 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-splunk-hec!
   minitest (~> 5.0)
+  net-smtp
   rake (>= 12.0)
   simplecov
   test-unit (~> 3.0)
   webmock (~> 3.5.0)
 
 BUNDLED WITH
-   2.3.9
+   2.3.20

data/README.md

@@ -1,13 +1,13 @@
 # fluent-plugin-splunk-hec
 
 [Fluentd](https://fluentd.org/) output plugin to send events and metrics to [Splunk](https://www.splunk.com) in 2 modes:<br/>
-1) Via Splunk's [HEC (HTTP Event Collector) API](http://dev.splunk.com/view/event-collector/SP-CAAAE7F)<br/> 
+1) Via Splunk's [HEC (HTTP Event Collector) API](http://dev.splunk.com/view/event-collector/SP-CAAAE7F)<br/>
 2) Via the Splunk Cloud Services (SCS) [Ingest API](https://sdc.splunkbeta.com/reference/api/ingest/v1beta2)
 
 ## Installation
 
 ### RubyGems
-``` 
+```
 $ gem install fluent-plugin-splunk-hec
 ```
 ### Bundler
@@ -157,7 +157,7 @@ This value must be set to `splunk_hec` when using HEC API and to `splunk_ingest_
 
 #### protocol (enum) (optional)
 
-This is the protocol to use for calling the HEC API. Available values are: http, https. This parameter is 
+This is the protocol to use for calling the HEC API. Available values are: http, https. This parameter is
 set to `https` by default.
 
 ### hec_host (string) (required)
@@ -172,6 +172,10 @@ The port number for the HEC token or the HEC load balancer. The default value is
 
 Identifier for the HEC token.
 
+### hec_endpoint (string) (optional)
+
+The HEC REST API endpoint to use. The default value is `services/collector`.
+
 ### metrics_from_event (bool) (optional)
 
 When `data_type` is set to "metric", the ingest API will treat every key-value pair in the input event as a metric name-value pair. Set `metrics_from_event` to `false` to disable this behavior and use `metric_name_key` and `metric_value_key` to define metrics. The default value is `true`.
@@ -194,31 +198,31 @@ If `coerce_to_utf8` is set to `true`, any non-UTF-8 character is replaced by the
 
 ### Parameters for `splunk_ingest_api`
 
-### service_client_identifier: (optional) (string) 
+### service_client_identifier: (optional) (string)
 
 Splunk uses the client identifier to make authorized requests to the ingest API.
 
-### service_client_secret_key: (string) 
+### service_client_secret_key: (string)
 
 The client identifier uses this authorization to make requests to the ingest API.
 
-### token_endpoint: (string) 
+### token_endpoint: (string)
 
 This value indicates which endpoint Splunk should look to for the authorization token necessary for requests to the ingest API.
 
-### ingest_api_host: (string) 
+### ingest_api_host: (string)
 
 Indicates which url/hostname to use for requests to the ingest API.
 
-### ingest_api_tenant: (string) 
+### ingest_api_tenant: (string)
 
 Indicates which tenant Splunk should use for requests to the ingest API.
 
-### ingest_api_events_endpoint: (string) 
+### ingest_api_events_endpoint: (string)
 
 Indicates which endpoint to use for requests to the ingest API.
 
-### debug_http: (bool) 
+### debug_http: (bool)
 Set to True if you want to debug requests and responses to ingest API. Default is false.
 
 ### Parameters for both `splunk_hec` and `splunk_ingest_api`
@@ -330,7 +334,7 @@ If a parameter has just a key, it means its value is exactly the same as the key
 
 #### When `data_type` is `metric`
 
-For metrics, parameters inside `<fields>` are used as dimensions. If `<fields>` is not presented, the original input event will be used as dimensions. If an empty `<fields></fields>` is presented, no dimension is sent. For example, given the following configuration: 
+For metrics, parameters inside `<fields>` are used as dimensions. If `<fields>` is not presented, the original input event will be used as dimensions. If an empty `<fields></fields>` is presented, no dimension is sent. For example, given the following configuration:
 
 ```
 <match **>
@@ -381,7 +385,7 @@ Multiple `<format>` sections can be defined to use different formatters for diff
   </format>
 ```
 
-This example: 
+This example:
 - Formats events with tags that start with `sometag.` with the `single_value` formatter
 - Formats events with tags `some.othertag` with the `csv` formatter
 - Formats all other events with the `json` formatter (the default formatter)
@@ -398,9 +402,10 @@ The following parameters can be used for tuning HTTP connections:
 
 #### idle_timeout (integer)
 
-The default is five seconds. If a connection has not been used for five seconds, it is automatically reset at next use, in order to avoid attempting to send to a closed connection. Specifiy `nil` to prohibit any timeouts. 
+The default is five seconds. If a connection has not been used for five seconds, it is automatically reset at next use, in order to avoid attempting to send to a closed connection. Specifiy `nil` to prohibit any timeouts.
 
 #### read_timeout (integer)
+
 The amount of time allowed between reading two chunks from the socket. The default value is `nil`, which means no timeout. 
 
 #### open_timeout (integer)
@@ -421,11 +426,11 @@ The private key for this client.
 
 #### ca_file (string)
 
-The path to a file containing a PEM-format CA certificate.
+The path to a file containing CA cerificates in PEM format. The plugin will verify the TLS server certificate presented by Splunk against the certificates in this file, unless verification is disabled by the `ssl_insecure` option.
 
 #### ca_path (string)
 
-The path to a directory containing CA certificates in PEM format.
+The path to a directory containing CA certificates in PEM format. The plugin will verify the TLS server certificate presented by Splunk against the certificates in this file, unless verification is disabled by the `ssl_insecure` option.
 
 #### ciphers (array)
 
@@ -433,15 +438,15 @@ List of SSl ciphers allowed.
 
 #### insecure_ssl (bool)
 
-Specifies whether an insecure SSL connection is allowed. If set to false, Splunk does not verify an insecure server certificate. This parameter is set to `false` by default. Ensure parameter `ca_file` is not configured in order to allow insecure SSL connections when this value is set to `true`.
+Specifies whether an insecure SSL connection is allowed. If set to `false` (the default), the plugin will verify the TLS server certificate presented by Splunk against the CA certificates provided by the `ca_file`/`ca_path` options, and reject the certificate if if verification fails.
 
 #### require_ssl_min_version (bool)
 
-When set to true, TLS version 1.1 and above is required.
+When set to `true` (the default), the plugin will require TLSv1.1 or later for its connection to Splunk.
 
 #### consume_chunk_on_4xx_errors (bool)
 
-Specifies whether any 4xx HTTP response status code consumes the buffer chunks. If set to false, Splunk will fail to flush the buffer on such status codes. This parameter is set to `true` by default for backwards compatibility.
+Specifies whether any 4xx HTTP response status code consumes the buffer chunks. If set to `false`, Splunk will fail to flush the buffer on such status codes. This parameter is set to `true` by default for backwards compatibility.
 
 ## About Buffer
 
@@ -456,4 +461,4 @@ Here are some hints:
 
 ## License
 
-Please see [LICENSE](LICENSE). 
+Please see [LICENSE](LICENSE).

data/VERSION

@@ -1 +1 @@
-1.2.11
+1.3.0

data/fluent-plugin-splunk-hec.gemspec

@@ -33,9 +33,9 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_runtime_dependency 'fluentd', '>= 1.4'
+  spec.add_runtime_dependency 'fluentd', '>= 1.5'
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
-  spec.add_runtime_dependency 'net-http-persistent', '~> 3.1'
+  spec.add_runtime_dependency 'net-http-persistent', '~> 4.0'
   spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'
   spec.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
 

data/lib/fluent/plugin/out_splunk.rb

@@ -92,6 +92,10 @@ module Fluent::Plugin
       end
     end
 
+    def shutdown
+      super
+    end
+
     def write(chunk)
       log.trace { "#{self.class}: Received new chunk, size=#{chunk.read.bytesize}" }
 

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -9,6 +9,7 @@ require 'fluent/plugin/out_splunk'
 require 'openssl'
 require 'multi_json'
 require 'net/http/persistent'
+require 'zlib'
 
 module Fluent::Plugin
   class SplunkHecOutput < SplunkOutput
@@ -36,6 +37,9 @@ module Fluent::Plugin
     desc 'The port number to HEC, or HEC load balancer.'
     config_param :hec_port, :integer, default: 8088
 
+    desc 'HEC REST API endpoint to use'
+    config_param :hec_endpoint, :string, default: 'services/collector'
+
     desc 'Full url to connect tosplunk. Example: https://mydomain.com:8088/apps/splunk'
     config_param :full_url, :string, default: ''
 
@@ -93,6 +97,9 @@ module Fluent::Plugin
     desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
     config_param :keep_keys, :bool, default: false
 
+    desc 'Indicates if GZIP Compression is enabled.'
+    config_param :gzip_compression, :bool, default: false
+
     desc 'App name'
     config_param :app_name, :string, default: "hec_plugin_gem"
 
@@ -166,8 +173,8 @@ module Fluent::Plugin
     end
 
     def shutdown
+      @conn.shutdown if not @conn.nil?
       super
-      @conn.shutdown
     end
 
     def format(tag, time, record)
@@ -287,9 +294,9 @@ module Fluent::Plugin
 
     def construct_api
       if @full_url.empty?
-        URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
+        URI("#{@protocol}://#{@hec_host}:#{@hec_port}/#{@hec_endpoint.delete_prefix("/")}")
       else
-        URI("#{@full_url.delete_suffix("/")}/services/collector")
+        URI("#{@full_url.delete_suffix("/")}/#{@hec_endpoint.delete_prefix("/")}")
       end
     rescue StandardError
       if @full_url.empty?
@@ -318,13 +325,20 @@ module Fluent::Plugin
         c.override_headers['Authorization'] = "Splunk #{@hec_token}"
         c.override_headers['__splunk_app_name'] = "#{@app_name}"
         c.override_headers['__splunk_app_version'] = "#{@app_version}"
-
       end
     end
 
     def write_to_splunk(chunk)
       post = Net::HTTP::Post.new @api.request_uri
-      post.body = chunk.read
+      if @gzip_compression
+        post.add_field("Content-Encoding", "gzip")
+        gzip_stream = Zlib::GzipWriter.new StringIO.new
+        gzip_stream << chunk.read
+        post.body = gzip_stream.close.string
+      else
+        post.body = chunk.read
+      end
+
       log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
       log.trace { "POST #{@api} body=#{post.body}" }
       begin

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -60,6 +60,12 @@ describe Fluent::Plugin::SplunkHecOutput do
     it 'should consume chunks on 4xx errors' do
       expect(create_hec_output_driver('hec_host hec_token').instance.consume_chunk_on_4xx_errors).must_equal true
     end
+    it 'should default gzip off' do
+      expect(create_hec_output_driver('hec_host hec_token').instance.gzip_compression).must_equal false
+    end
+    it 'should support enabling gzip' do
+      expect(create_hec_output_driver('hec_host hec_token', 'gzip_compression true').instance.gzip_compression).must_equal true
+    end
   end
 
   describe 'hec_host validation' do
@@ -357,6 +363,15 @@ describe Fluent::Plugin::SplunkHecOutput do
     end
   end
 
+  describe 'gzip encoding' do
+    it 'should include gzip header when enabled' do
+      metrics = [
+        ['tag', event_time, { 'cup': 0.5, 'memory': 100 }]
+      ]
+      with_stub_hec_gzip(events: metrics, conf: 'data_type metric')
+    end
+  end
+
   def with_stub_hec(events:, conf: '')
     host = 'hec.splunk.com'
     @driver = create_hec_output_driver("hec_host #{host}", conf)
@@ -372,6 +387,21 @@ describe Fluent::Plugin::SplunkHecOutput do
     hec_req
   end
 
+  def with_stub_hec_gzip(events:, conf: '')
+    host = 'hec.splunk.com'
+    @driver = create_hec_output_driver("hec_host #{host}", 'gzip_compression true', conf)
+
+    hec_req = stub_hec_gzip_request("https://#{host}:8088").with do |r|
+      yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }
+    end
+
+    @driver.run do
+      events.each { |evt| @driver.feed *evt }
+    end
+
+    hec_req
+  end
+
   def verify_sent_events(conf = '', &blk)
     event = {
       'log' => 'everything is good',

data/test/test_helper.rb

@@ -42,4 +42,15 @@ module PluginTestHelper
                        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" })
       .to_return(body: '{"text":"Success","code":0}')
   end
+
+  def stub_hec_gzip_request(endpoint)
+    stub_request(:post, "#{endpoint}/services/collector")
+      .with(headers: {
+        'Authorization' => "Splunk #{TEST_HEC_TOKEN}",
+        'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}",
+        'Content-Encoding' => "gzip"
+      },
+      )
+      .to_return(body: '{"text":"GzipSuccess","code":0}')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.2.11
+  version: 1.3.0
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-15 00:00:00.000000000 Z
+date: 2022-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: openid_connect
   requirement: !ruby/object:Gem::Requirement
@@ -221,13 +221,13 @@ signing_key:
 specification_version: 4
 summary: Fluentd plugin for Splunk HEC.
 test_files:
-- test/fluent/plugin/out_splunk_ingest_api_test.rb
-- test/fluent/plugin/out_splunk_hec_test.rb
 - test/test_helper.rb
 - test/lib/webmock/http_lib_adapters/excon_adapter.rb
-- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
 - test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
 - test/lib/webmock/http_lib_adapters/manticore_adapter.rb
 - test/lib/webmock/http_lib_adapters/curb_adapter.rb
-- test/lib/webmock/http_lib_adapters/patron_adapter.rb
 - test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/fluent/plugin/out_splunk_hec_test.rb
+- test/fluent/plugin/out_splunk_ingest_api_test.rb