fluent-plugin-splunk-hec 1.2.5 → 1.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 964d0f2e9840a7fa66a70dafb6918b3a54b7586e0ed550a9651ac5e150b817a2
-  data.tar.gz: 97ef4f8a6f602bdfc359b4119800837c40571d0b717ba3a358aa1a66fd8ca64e
+  metadata.gz: 3fbf155dc5649294d859dbdc4bc9967681f8d4051f93e7c108bf926961f93afe
+  data.tar.gz: 1bf76e9bb39b3bbf9dc0ae87f8ca38046cbe4f30a86ae0c6c6fef4be257fbb74
 SHA512:
-  metadata.gz: b74c2408af64d38611627af3e16a5723c32bbce47002e030ddc09b7793d4f72a42a933330554e5b5e4092d66f45564d7038bb96f1ae75131f70a1fa9c9a86acf
-  data.tar.gz: 324f4e82f7b3d8792798f44ec5647f56845590fd3bf317ad6737f19788c9483112464635417389b7cb024dc34c23edab0bf854ae086c54404a368c902d5eb420
+  metadata.gz: 5228076cb01aa5a32c77f7d532ecb7052f57366e2dbd9d5f1e3fcae1cb7d5ca042bec12393f4c760ce05c4205068473852fdf020255b45d20679af2ee623da85
+  data.tar.gz: 82a15c018c694133cb6e5d4891413ba272087f385586582ae5b00a91b78bc455918116ee078f52f5cc26f7440852c4f2549d1b1730f2c5f3480130896f7d6fe7

data/Gemfile.lock

@@ -1,60 +1,59 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-splunk-hec (1.2.5)
+    fluent-plugin-splunk-hec (1.2.9)
       fluentd (>= 1.4)
       multi_json (~> 1.13)
       net-http-persistent (~> 3.1)
       openid_connect (~> 1.1.8)
-      prometheus-client (< 0.10.0)
+      prometheus-client (>= 2.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.3)
-      activesupport (= 6.1.3)
-    activesupport (6.1.3)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aes_key_wrap (1.1.0)
-    ast (2.4.2)
     attr_required (1.0.1)
-    bindata (2.4.8)
-    concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
+    bindata (2.4.10)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
     cool.io (1.7.1)
     crack (0.4.5)
       rexml
-    docile (1.3.5)
-    fluentd (1.12.1)
+    docile (1.4.0)
+    fluentd (1.14.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
     hashdiff (1.0.1)
-    http_parser.rb (0.6.0)
+    http_parser.rb (0.7.0)
     httpclient (2.8.3)
-    i18n (1.8.9)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     json-jwt (1.13.0)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    mini_mime (1.0.2)
+    mini_mime (1.1.2)
     minitest (5.14.4)
     msgpack (1.4.2)
     multi_json (1.15.0)
@@ -70,35 +69,19 @@ GEM
       validate_email
       validate_url
       webfinger (>= 1.0.1)
-    parallel (1.20.1)
-    parser (3.0.0.0)
-      ast (~> 2.4.1)
-    power_assert (2.0.0)
-    powerpack (0.1.3)
-    prometheus-client (0.9.0)
-      quantile (~> 0.2.1)
+    power_assert (2.0.1)
+    prometheus-client (2.1.0)
     public_suffix (4.0.6)
-    quantile (0.2.1)
     rack (2.2.3)
-    rack-oauth2 (1.16.0)
+    rack-oauth2 (1.19.0)
       activesupport
       attr_required
       httpclient
       json-jwt (>= 1.11.0)
       rack (>= 2.1.0)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    rexml (3.2.4)
-    rubocop (0.63.1)
-      jaro_winkler (~> 1.5.1)
-      parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      powerpack (~> 0.1)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.4.0)
-    ruby-progressbar (1.11.0)
-    serverengine (2.2.3)
+    rake (13.0.6)
+    rexml (3.2.5)
+    serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -106,34 +89,34 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     strptime (0.2.5)
-    swd (1.2.0)
+    swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
       httpclient (>= 2.4)
-    test-unit (3.4.0)
+    test-unit (3.5.0)
       power_assert
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
-    unicode-display_width (1.4.1)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
     validate_url (1.0.13)
       activemodel (>= 3.0.0)
       public_suffix
-    webfinger (1.1.0)
+    webfinger (1.2.0)
       activesupport
       httpclient (>= 2.4)
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
+    webrick (1.7.0)
     yajl-ruby (1.4.1)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   ruby
@@ -143,10 +126,9 @@ DEPENDENCIES
   fluent-plugin-splunk-hec!
   minitest (~> 5.0)
   rake (>= 12.0)
-  rubocop (~> 0.63.1)
   simplecov
   test-unit (~> 3.0)
   webmock (~> 3.5.0)
 
 BUNDLED WITH
-   2.2.15
+   2.2.33

data/LICENSE

@@ -270,7 +270,6 @@ The following components are provided under the MIT License. See project link fo
     (MIT License) rake (https://github.com/ruby/rake/blob/master/MIT-LICENSE)
     (MIT License) recursive-open-struct (https://github.com/aetherknight/recursive-open-struct/blob/master/LICENSE.txt)
     (MIT License) rest-client (https://github.com/rest-client/rest-client/blob/master/LICENSE)
-    (MIT License) rubocop (https://github.com/rubocop-hq/rubocop/blob/master/LICENSE.txt)
     (MIT License) ruby-progressbar (https://github.com/jfelchner/ruby-progressbar/blob/master/LICENSE.txt)
     (MIT License) safe_yaml (https://github.com/dtao/safe_yaml/blob/master/LICENSE.txt)
     (MIT License) sigdump (https://github.com/frsyuki/sigdump/blob/master/LICENSE)

data/README.md

@@ -1,4 +1,3 @@
-[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-splunk-hec)
 # fluent-plugin-splunk-hec
 
 [Fluentd](https://fluentd.org/) output plugin to send events and metrics to [Splunk](https://www.splunk.com) in 2 modes:<br/>

data/Rakefile

@@ -2,9 +2,6 @@
 
 require 'bundler/gem_tasks'
 require 'rake/testtask'
-require 'rubocop/rake_task'
-
-RuboCop::RakeTask.new
 
 Rake::TestTask.new(:test) do |t|
   t.libs << 'test'

data/VERSION

@@ -1 +1 @@
-1.2.5
+1.2.9

data/fluent-plugin-splunk-hec.gemspec

@@ -37,13 +37,12 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
   spec.add_runtime_dependency 'net-http-persistent', '~> 3.1'
   spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'
-  spec.add_runtime_dependency 'prometheus-client', '< 0.10.0'
+  spec.add_runtime_dependency 'prometheus-client', '>= 2.1.0'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '>= 12.0'
   # required by fluent/test.rb
   spec.add_development_dependency 'minitest', '~> 5.0'
-  spec.add_development_dependency 'rubocop', '~> 0.63.1'
   spec.add_development_dependency 'simplecov', '~> 0.16.1'
   spec.add_development_dependency 'test-unit', '~> 3.0'
   spec.add_development_dependency 'webmock', '~> 3.5.0'

data/lib/fluent/plugin/out_splunk.rb

@@ -100,11 +100,11 @@ module Fluent::Plugin
         write_to_splunk(chunk)
       end
 
-      @metrics[:record_counter].increment(metric_labels, chunk.size_of_events)
-      @metrics[:bytes_counter].increment(metric_labels, chunk.bytesize)
-      @metrics[:write_records_histogram].observe(metric_labels, chunk.size_of_events)
-      @metrics[:write_bytes_histogram].observe(metric_labels, chunk.bytesize)
-      @metrics[:write_latency_histogram].observe(metric_labels, t)
+      @metrics[:record_counter].increment(labels: metric_labels, by: chunk.size_of_events)
+      @metrics[:bytes_counter].increment(labels: metric_labels, by: chunk.bytesize)
+      @metrics[:write_records_histogram].observe(chunk.size_of_events, labels: metric_labels)
+      @metrics[:write_bytes_histogram].observe(chunk.bytesize, labels: metric_labels, )
+      @metrics[:write_latency_histogram].observe(t, labels: metric_labels, )
     end
 
     def write_to_splunk(_chunk)
@@ -153,7 +153,7 @@ module Fluent::Plugin
     def process_response(response, _request_body)
       log.trace { "[Response] POST #{@api}: #{response.inspect}" }
 
-      @metrics[:status_counter].increment(metric_labels(status: response.code.to_s))
+      @metrics[:status_counter].increment(labels: metric_labels(status: response.code.to_s))
 
       raise_err = response.code.to_s.start_with?('5') || (!@consume_chunk_on_4xx_errors && response.code.to_s.start_with?('4'))
 
@@ -193,8 +193,8 @@ module Fluent::Plugin
           v = instance_variable_get "@#{f}"
           next unless v
 
-          if v == TAG_PLACEHOLDER
-            instance_variable_set "@#{f}", ->(tag, _) { tag }
+          if v.include? TAG_PLACEHOLDER
+            instance_variable_set "@#{f}", ->(tag, _) { v.gsub(TAG_PLACEHOLDER, tag) }
           else
             instance_variable_set "@#{f}", ->(_, _) { v }
           end
@@ -209,7 +209,7 @@ module Fluent::Plugin
       # This loop looks dump, but it is used to suppress the unused parameter configuration warning
       # Learned from `filter_record_transformer`.
       conf.elements.select { |element| element.name == 'fields' }.each do |element|
-        element.each_pair { |k, _v| element.key?(k) }
+        element.each_pair { |k, _v| element.has_key?(k) }
       end
 
       return unless @fields
@@ -235,28 +235,34 @@ module Fluent::Plugin
 
       @metrics = {
         record_counter: register_metric(::Prometheus::Client::Counter.new(
-                                          :splunk_output_write_records_count,
-                                          'The number of log records being sent'
+                                          :splunk_output_write_records_count, docstring:
+                                          'The number of log records being sent',
+                                          labels: metric_label_keys
                                         )),
         bytes_counter: register_metric(::Prometheus::Client::Counter.new(
-                                         :splunk_output_write_bytes_count,
-                                         'The number of log bytes being sent'
+                                         :splunk_output_write_bytes_count, docstring:
+                                         'The number of log bytes being sent',
+                                         labels: metric_label_keys
                                        )),
         status_counter: register_metric(::Prometheus::Client::Counter.new(
-                                          :splunk_output_write_status_count,
-                                          'The count of sends by response_code'
+                                          :splunk_output_write_status_count, docstring:
+                                          'The count of sends by response_code',
+                                          labels: metric_label_keys(status: "")
                                         )),
         write_bytes_histogram: register_metric(::Prometheus::Client::Histogram.new(
-                                                 :splunk_output_write_payload_bytes,
-                                                 'The size of the write payload in bytes', {}, [1024, 23_937, 47_875, 95_750, 191_500, 383_000, 766_000, 1_149_000]
+                                                 :splunk_output_write_payload_bytes, docstring:
+                                                 'The size of the write payload in bytes', buckets: [1024, 23_937, 47_875, 95_750, 191_500, 383_000, 766_000, 1_149_000],
+                                                 labels: metric_label_keys
                                                )),
         write_records_histogram: register_metric(::Prometheus::Client::Histogram.new(
-                                                   :splunk_output_write_payload_records,
-                                                   'The number of records written per write', {}, [1, 10, 25, 100, 200, 300, 500, 750, 1000, 1500]
+                                                   :splunk_output_write_payload_records, docstring:
+                                                   'The number of records written per write', buckets: [1, 10, 25, 100, 200, 300, 500, 750, 1000, 1500],
+                                                   labels: metric_label_keys
                                                  )),
         write_latency_histogram: register_metric(::Prometheus::Client::Histogram.new(
-                                                   :splunk_output_write_latency_seconds,
-                                                   'The latency of writes'
+                                                   :splunk_output_write_latency_seconds, docstring:
+                                                   'The latency of writes',
+                                                   labels: metric_label_keys
                                                  ))
       }
     end
@@ -266,6 +272,10 @@ module Fluent::Plugin
       @metric_labels.merge other_labels
     end
 
+    def metric_label_keys(other_labels = {})
+      (@metric_labels.merge other_labels).keys
+    end
+
     # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
     # non-UTF-8 character would be replaced by the string specified by
     # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -31,13 +31,16 @@ module Fluent::Plugin
     config_param :protocol, :enum, list: %i[http https], default: :https
 
     desc 'The hostname/IP to HEC, or HEC load balancer.'
-    config_param :hec_host, :string
+    config_param :hec_host, :string, default: ''
 
     desc 'The port number to HEC, or HEC load balancer.'
     config_param :hec_port, :integer, default: 8088
 
+    desc 'Full url to connect tosplunk. Example: https://mydomain.com:8088/apps/splunk'
+    config_param :full_url, :string, default: ''
+
     desc 'The HEC token.'
-    config_param :hec_token, :string
+    config_param :hec_token, :string, secret: true
 
     desc 'If a connection has not been used for this number of seconds it will automatically be reset upon the next use to avoid attempting to send to a closed connection. nil means no timeout.'
     config_param :idle_timeout, :integer, default: 5
@@ -132,7 +135,7 @@ module Fluent::Plugin
 
     def configure(conf)
       super
-
+      raise Fluent::ConfigError, 'One of `hec_host` or `full_url` is required.' if @hec_host.empty? && @full_url.empty?
       check_metric_configs
       pick_custom_format_method
     end
@@ -279,9 +282,17 @@ module Fluent::Plugin
     end
 
     def construct_api
-      URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
+      if @full_url.empty?
+        URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
+      else
+        URI("#{@full_url.delete_suffix("/")}/services/collector")
+      end
     rescue StandardError
-      raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid."
+      if @full_url.empty?
+        raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid."
+      else
+        raise Fluent::ConfigError, "full_url (#{@full_url}) is invalid."
+      end
     end
 
     def new_connection
@@ -312,10 +323,13 @@ module Fluent::Plugin
       post.body = chunk.read
       log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
       log.trace { "POST #{@api} body=#{post.body}" }
-
-      t1 = Time.now
-      response = @conn.request @api, post
-      t2 = Time.now
+      begin
+        t1 = Time.now
+        response = @conn.request @api, post
+        t2 = Time.now
+      rescue Net::HTTP::Persistent::Error => e
+        raise e.cause
+      end
 
       raise_err = response.code.to_s.start_with?('5') || (!@consume_chunk_on_4xx_errors && response.code.to_s.start_with?('4'))
 

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -64,7 +64,7 @@ describe Fluent::Plugin::SplunkHecOutput do
 
   describe 'hec_host validation' do
     describe 'invalid host' do
-      it 'should require hec_host' do
+      it 'should require hec_host or full_url' do
         expect { create_hec_output_driver }.must_raise Fluent::ConfigError
       end
 
@@ -78,6 +78,17 @@ describe Fluent::Plugin::SplunkHecOutput do
     end
   end
 
+  describe 'full_url validation' do
+    describe 'invalid full_url' do
+      it { expect { create_hec_output_driver(full_url: '%bad-host%.com') }.must_raise Fluent::ConfigError }
+    end
+    describe 'good full_url' do
+      it { 
+        expect(create_hec_output_driver('full_url https://splunk.com').instance.full_url).must_equal 'https://splunk.com'
+      }
+    end
+  end
+
   it 'should send request to Splunk' do
     req = verify_sent_events do |batch|
       expect(batch.size).must_equal 2

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.9
 platform: ruby
 authors:
 - Splunk Inc.
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-29 00:00:00.000000000 Z
+date: 2021-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -70,16 +70,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +122,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.63.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.63.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -215,7 +201,7 @@ homepage: https://github.com/splunk/fluent-plugin-splunk-hec
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -230,18 +216,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Splunk HEC.
 test_files:
-- test/fluent/plugin/out_splunk_hec_test.rb
-- test/fluent/plugin/out_splunk_ingest_api_test.rb
-- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/test_helper.rb
 - test/lib/webmock/http_lib_adapters/manticore_adapter.rb
-- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
-- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
-- test/lib/webmock/http_lib_adapters/curb_adapter.rb
 - test/lib/webmock/http_lib_adapters/http_rb_adapter.rb
+- test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb
 - test/lib/webmock/http_lib_adapters/excon_adapter.rb
-- test/test_helper.rb
+- test/lib/webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/webmock/http_lib_adapters/patron_adapter.rb
+- test/fluent/plugin/out_splunk_ingest_api_test.rb
+- test/fluent/plugin/out_splunk_hec_test.rb