fluent-plugin-sumologic_output 1.7.3 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52b0c77e900d61ba360421933440383208033b29b0f08b012250519189872f24
-  data.tar.gz: ff86f952744e650e30ba94d1185f4d270a3d8d6e7bfee5f30a0810e043a9483f
+  metadata.gz: 280c08203277770f0dc9277d9df2f8249fba58283816e4fd5b511560fb533ac3
+  data.tar.gz: ad2b07fadad2698bf9169a0556a3c63422446267c292c27c71e0d5a8e7b5669a
 SHA512:
-  metadata.gz: 74c43d262f35e2767d170a05dbec58911590c0851a724f6c4adb0047a26601383e4d8fa5e2671c845bbd89a65781fb0d35e4bc2f8d7ed6bd0f0e376d6c7ec958
-  data.tar.gz: 73b8ea2a6dbf4593eca7d4a702eeea14f27728bd7aa28ddc15e968301a5a2eb21f76ae18f1be2dd759f42cb357c2ceda92aec49e136755d76c954ebded889cdd
+  metadata.gz: 32636372bdbb936f6d3c68a223b1cdd4b9e716522fd526df80b079f1712c77f658f16c3f60d5554f80d53354006e8d11e40a9fbe68c0b4fbd767fe053a9cc8bb
+  data.tar.gz: 76b52ddad11c35a1e2be4a6f818e6f2f60f10ad6a74147c43fb05f2689dbc4603c9429437ff6c7cdd9a1cdf398894096adb925f1dd922b117bf509ba12c8ffa4

data/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.8.0"></a>
+# [1.8.0] (2022-04-22)
+
+- feat: add exponential backoff for sending data [#76](https://github.com/SumoLogic/fluentd-output-sumologic/pull/76)
+- feat(max_request_size): add max_request_size to limit size of requests [#78](https://github.com/SumoLogic/fluentd-output-sumologic/pull/78)
+
+<a name="1.7.5"></a>
+# [1.7.5] (2022-04-11)
+
+- refactor: add a debug log on sending [#75](https://github.com/SumoLogic/fluentd-output-sumologic/pull/75)
+
+<a name="1.7.4"></a>
+# [1.7.4] (2021-04-08)
+
+- fix: handle receiver warning messages [#73](https://github.com/SumoLogic/fluentd-output-sumologic/pull/73)
+
+<a name="1.7.3"></a>
+# [1.7.3] (2021-10-19)
+- Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)
+- Fix json parsing [#69](https://github.com/SumoLogic/fluentd-output-sumologic/pull/69)
+
+<a name="1.7.2"></a>
+# [1.7.2] (2020-11-23)
+- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
+
 <a name="1.7.1"></a>
 # [1.7.1] (2020-04-28)
 - Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)

data/README.md

@@ -38,6 +38,13 @@ Configuration options for fluent.conf are:
 * `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
 * `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
 * `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
+* `use_internal_retry` - Enable custom retry mechanism. As this is `false` by default due to backward compatibility,
+  we recommend to enable it and configure the following parameters (`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times`)
+* `retry_min_interval` - Minimum interval to wait between sending tries (default is `1s`)
+* `retry_max_interval` - Maximum interval to wait between sending tries (default is `5m`)
+* `retry_timeout` - Time after which the data is going to be dropped (default is `72h`) (`0s` means that there is no timeout)
+* `retry_max_times` - Maximum number of retries (default is `0`) (`0` means that there is no max retry times, retries will happen forever)
+* `max_request_size` - Maximum request size (before applying compression). Default is `0k` which means no limit
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 
@@ -137,6 +144,14 @@ Example
 }
 ```
 
+## Retry Mechanism
+
+`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times` are not the [buffer retries parameters][buffer_retries].
+Due to technical reason, this plugin implements it's own retrying back-off exponential mechanism.
+It is disabled by default, but we recommend to enable it by setting `use_internal_retry` to `true`.
+
+[buffer_retries]: https://docs.fluentd.org/configuration/buffer-section#retries-parameters
+
 ### TLS 1.2 Requirement
 
 Sumo Logic only accepts connections from clients using TLS version 1.2 or greater. To utilize the content of this repo, ensure that it's running in an execution environment that is configured to use TLS 1.2 or greater.

data/Vagrantfile

@@ -12,7 +12,9 @@ Vagrant.configure('2') do |config|
   config.disksize.size = '50GB'
   config.vm.box_check_update = false
   config.vm.host_name = 'fluentd-output-sumologic'
-  config.vm.network :private_network, ip: "192.168.78.45"
+  # Vbox 6.1.28+ restricts host-only adapters to 192.168.56.0/21
+  # See: https://www.virtualbox.org/manual/ch06.html#network_hostonly
+  config.vm.network :private_network, ip: "192.168.56.45"
 
   config.vm.provider 'virtualbox' do |vb|
     vb.gui = false

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.3"
+  gem.version       = "1.8.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -13,12 +13,13 @@ class SumologicConnection
   COMPRESS_DEFLATE = 'deflate'
   COMPRESS_GZIP = 'gzip'
 
-  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
+  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding, logger)
     @endpoint = endpoint
     @sumo_client = sumo_client
     create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @compress = compress_enabled
     @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+    @logger = logger
 
     unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
       raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
@@ -30,6 +31,29 @@ class SumologicConnection
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
+
+    # response is 20x, check response content
+    return if response.content.length == 0
+    
+    # if we get a non-empty response, check it 
+    begin
+      response_map = JSON.load(response.content)
+    rescue JSON::ParserError
+      @logger.warn "Error decoding receiver response: #{response.content}"
+      return
+    end
+
+    # log a warning with the present keys
+    response_keys = ["id", "code", "status", "message", "errors"]
+    log_params = []
+    response_keys.each do |key|
+      if response_map.has_key?(key) then
+        value = response_map[key]
+        log_params.append("#{key}: #{value}")
+      end
+    end
+    log_params_str = log_params.join(", ")
+    @logger.warn "There was an issue sending data: #{log_params_str}"
   end
 
   def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
@@ -133,6 +157,15 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+
+  config_param :use_internal_retry, :bool, :default => false
+  config_param :retry_timeout, :time, :default => 72 * 3600  # 72h
+  config_param :retry_max_times, :integer, :default => 0
+  config_param :retry_min_interval, :time, :default => 1  # 1s
+  config_param :retry_max_interval, :time, :default => 5*60  # 5m
+
+  config_param :max_request_size, :size, :default => 0
+
   # https://help.sumologic.com/Manage/Fields
   desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
   config_param :custom_fields, :string, :default => nil
@@ -218,8 +251,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       conf['disable_cookies'],
       conf['sumo_client'],
       conf['compress'],
-      conf['compress_encoding']
+      conf['compress_encoding'],
+      log,
       )
+
+    if !conf['max_request_size'].nil? && conf['max_request_size'].to_i <= 0
+      conf['max_request_size'] = '0'
+    end
     super
   end
 
@@ -360,6 +398,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     end
 
+    chunk_id = "##{chunk.dump_unique_id_hex(chunk.unique_id)}"
     # Push logs to sumo
     messages_list.each do |key, messages|
       source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
@@ -372,16 +411,78 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
         fields = [fields,@custom_fields].compact.join(",")
       end
 
-      @sumo_conn.publish(
-          messages.join("\n"),
-          source_host         =source_host,
-          source_category     =source_category,
-          source_name         =source_name,
-          data_type           =@data_type,
-          metric_data_format  =@metric_data_format,
-          collected_fields    =fields,
-          dimensions          =@custom_dimensions
-      )
+      if @max_request_size <= 0
+        messages_to_send = [messages]
+      else
+        messages_to_send = []
+        current_message = []
+        current_length = 0
+        messages.each do |message|
+          current_message.push message
+          current_length += message.length
+
+          if current_length > @max_request_size
+            messages_to_send.push(current_message)
+            current_message = []
+            current_length = 0
+          end
+          current_length += 1  # this is for newline
+        end
+        if current_message.length > 0
+          messages_to_send.push(current_message)
+        end
+      end
+      
+      messages_to_send.each_with_index do |message, i|
+        retries = 0
+        start_time = Time.now
+        sleep_time = @retry_min_interval
+
+        while true
+          common_log_part = "#{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}', chunk #{chunk_id}, try #{retries}, batch #{i}"
+
+          begin
+            @log.debug { "Sending #{message.count}; #{common_log_part}" }
+
+            @sumo_conn.publish(
+              message.join("\n"),
+                source_host         =source_host,
+                source_category     =source_category,
+                source_name         =source_name,
+                data_type           =@data_type,
+                metric_data_format  =@metric_data_format,
+                collected_fields    =fields,
+                dimensions          =@custom_dimensions
+            )
+            break
+          rescue => e
+            if !@use_internal_retry
+              raise e
+            end
+            # increment retries
+            retries += 1
+
+            log.warn "error while sending request to sumo: #{e}; #{common_log_part}"
+            log.warn_backtrace e.backtrace
+
+            # drop data if
+            #   - we reached out the @retry_max_times retries
+            #   - or we exceeded @retry_timeout
+            if (retries >= @retry_max_times && @retry_max_times > 0) || (Time.now > start_time + @retry_timeout && @retry_timeout > 0)
+              log.warn "dropping records; #{common_log_part}"
+              break
+            end
+
+            log.info "going to retry to send data at #{Time.now + sleep_time}; #{common_log_part}"
+            sleep sleep_time
+
+            sleep_time *= 2
+            if sleep_time > @retry_max_interval
+              sleep_time = @retry_max_interval
+            end
+          end
+        end
+      end
     end
 
   end

data/test/plugin/test_out_sumologic.rb

@@ -795,6 +795,187 @@ class SumologicOutput < Test::Unit::TestCase
                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
                      body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
                      times:1
-  end  
+  end
+
+  def test_warning_response_from_receiver
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+    }
+    testdata = [
+      [
+        '{"id":"1TIRY-KGIVX-TPQRJ","errors":[{"code":"internal.error","message":"Internal server error."}]}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRJ, errors: [{"code"=>"internal.error", "message"=>"Internal server error."}]'
+      ],
+      [
+        '{"id":"1TIRY-KGIVX-TPQRX","code": 200, "status": "Fields dropped", "message": "Dropped fields above the 30 field limit"}', 
+        'There was an issue sending data: id: 1TIRY-KGIVX-TPQRX, code: 200, status: Fields dropped, message: Dropped fields above the 30 field limit'
+      ],
+    ]
+    time = event_time
+
+    testdata.each do |data, log|
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(body: data, headers: {content_type: 'application/json'})
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+      assert_equal driver.logs.length, 1
+      assert driver.logs[0].end_with?(log + "\n")
+    end
+  end
+
+  def test_resend
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 3
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      {status: 500, headers: {content_type: 'application/json'}},
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:2
+  end
+
+  def test_resend_failed
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 15
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      status: 500, headers: {content_type: 'application/json'}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:15
+  end
+
+  def test_resend_forever
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 0
+      retry_timeout 0s
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      *[{status: 500, headers: {content_type: 'application/json'}}]*123,
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:124
+  end
+
+  def test_skip_retry
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 500, headers: {content_type: 'application/json'})
+
+    exception = assert_raise(RuntimeError) {
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+    }
+    assert_equal("Failed to send data to HTTP Source. 500 - ", exception.message)
+  end
+
+  def test_split_negative_or_zero
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    configs = [
+      %{
+        endpoint #{endpoint}
+        max_request_size -5
+      },
+      %{
+        endpoint #{endpoint}
+        max_request_size 0
+      }
+    ]
+
+    time = event_time
+
+    configs.each do |config|
+      WebMock.reset_executed_requests!
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+      end
+
+      assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      body: /\A{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\z/,
+                      times:1
+    end
+  end
+
+  def test_split
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    config = %{
+        endpoint #{endpoint}
+        max_request_size 80
+      }
+
+    time = event_time
+
+    WebMock.reset_executed_requests!
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+    driver.run do
+      driver.feed("test", time, {"message": "test1"})
+      driver.feed("test", time, {"message": "test2"})
+      driver.feed("test", time, {"message": "test3"})
+      driver.feed("test", time, {"message": "test4"})
+      driver.feed("test", time, {"message": "test5"})
+    end
+
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test1"}\n{"timestamp":\d+.,"message":"test2"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test3"}\n{"timestamp":\d+.,"message":"test4"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test5"}\z/,
+                    times:1
+  end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.3
+  version: 1.8.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-19 00:00:00.000000000 Z
+date: 2022-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler