fluent-plugin-sumologic_output 1.7.5 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10221368588a63fe0f97cb60695466a78a12eae80c13f736b5ae54879e4a5b4e
-  data.tar.gz: 2b99181908e9df7b2bcae1e6534a9311d72464cb74923a77f3a7489eeeb13f94
+  metadata.gz: 280c08203277770f0dc9277d9df2f8249fba58283816e4fd5b511560fb533ac3
+  data.tar.gz: ad2b07fadad2698bf9169a0556a3c63422446267c292c27c71e0d5a8e7b5669a
 SHA512:
-  metadata.gz: 245aa88c7c415099db251eabf2c6b9e80a64d657cdf4df90e365115e962cbeec76279cab1d4d5dc87de1b6387475c0e6363a2be24c55d21953d4bc852727d869
-  data.tar.gz: c9a870b1252ca73b2565a4627e29ee5501ce9da6caa88834932d0edee05e45f573a5f7aeae183968e303786615622487188287236b1d12d8234ac992f25e822a
+  metadata.gz: 32636372bdbb936f6d3c68a223b1cdd4b9e716522fd526df80b079f1712c77f658f16c3f60d5554f80d53354006e8d11e40a9fbe68c0b4fbd767fe053a9cc8bb
+  data.tar.gz: 76b52ddad11c35a1e2be4a6f818e6f2f60f10ad6a74147c43fb05f2689dbc4603c9429437ff6c7cdd9a1cdf398894096adb925f1dd922b117bf509ba12c8ffa4

data/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.8.0"></a>
+# [1.8.0] (2022-04-22)
+
+- feat: add exponential backoff for sending data [#76](https://github.com/SumoLogic/fluentd-output-sumologic/pull/76)
+- feat(max_request_size): add max_request_size to limit size of requests [#78](https://github.com/SumoLogic/fluentd-output-sumologic/pull/78)
+
+<a name="1.7.5"></a>
+# [1.7.5] (2022-04-11)
+
+- refactor: add a debug log on sending [#75](https://github.com/SumoLogic/fluentd-output-sumologic/pull/75)
+
+<a name="1.7.4"></a>
+# [1.7.4] (2021-04-08)
+
+- fix: handle receiver warning messages [#73](https://github.com/SumoLogic/fluentd-output-sumologic/pull/73)
+
 <a name="1.7.3"></a>
 # [1.7.3] (2021-10-19)
 - Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)

data/README.md

@@ -38,6 +38,13 @@ Configuration options for fluent.conf are:
 * `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
 * `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
 * `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
+* `use_internal_retry` - Enable custom retry mechanism. As this is `false` by default due to backward compatibility,
+  we recommend to enable it and configure the following parameters (`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times`)
+* `retry_min_interval` - Minimum interval to wait between sending tries (default is `1s`)
+* `retry_max_interval` - Maximum interval to wait between sending tries (default is `5m`)
+* `retry_timeout` - Time after which the data is going to be dropped (default is `72h`) (`0s` means that there is no timeout)
+* `retry_max_times` - Maximum number of retries (default is `0`) (`0` means that there is no max retry times, retries will happen forever)
+* `max_request_size` - Maximum request size (before applying compression). Default is `0k` which means no limit
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 
@@ -137,6 +144,14 @@ Example
 }
 ```
 
+## Retry Mechanism
+
+`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times` are not the [buffer retries parameters][buffer_retries].
+Due to technical reason, this plugin implements it's own retrying back-off exponential mechanism.
+It is disabled by default, but we recommend to enable it by setting `use_internal_retry` to `true`.
+
+[buffer_retries]: https://docs.fluentd.org/configuration/buffer-section#retries-parameters
+
 ### TLS 1.2 Requirement
 
 Sumo Logic only accepts connections from clients using TLS version 1.2 or greater. To utilize the content of this repo, ensure that it's running in an execution environment that is configured to use TLS 1.2 or greater.

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.5"
+  gem.version       = "1.8.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -157,6 +157,15 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+
+  config_param :use_internal_retry, :bool, :default => false
+  config_param :retry_timeout, :time, :default => 72 * 3600  # 72h
+  config_param :retry_max_times, :integer, :default => 0
+  config_param :retry_min_interval, :time, :default => 1  # 1s
+  config_param :retry_max_interval, :time, :default => 5*60  # 5m
+
+  config_param :max_request_size, :size, :default => 0
+
   # https://help.sumologic.com/Manage/Fields
   desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
   config_param :custom_fields, :string, :default => nil
@@ -245,6 +254,10 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       conf['compress_encoding'],
       log,
       )
+
+    if !conf['max_request_size'].nil? && conf['max_request_size'].to_i <= 0
+      conf['max_request_size'] = '0'
+    end
     super
   end
 
@@ -385,6 +398,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     end
 
+    chunk_id = "##{chunk.dump_unique_id_hex(chunk.unique_id)}"
     # Push logs to sumo
     messages_list.each do |key, messages|
       source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
@@ -397,18 +411,78 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
         fields = [fields,@custom_fields].compact.join(",")
       end
 
-      @log.debug { "Sending #{messages.count} #{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}'." }
-
-      @sumo_conn.publish(
-          messages.join("\n"),
-          source_host         =source_host,
-          source_category     =source_category,
-          source_name         =source_name,
-          data_type           =@data_type,
-          metric_data_format  =@metric_data_format,
-          collected_fields    =fields,
-          dimensions          =@custom_dimensions
-      )
+      if @max_request_size <= 0
+        messages_to_send = [messages]
+      else
+        messages_to_send = []
+        current_message = []
+        current_length = 0
+        messages.each do |message|
+          current_message.push message
+          current_length += message.length
+
+          if current_length > @max_request_size
+            messages_to_send.push(current_message)
+            current_message = []
+            current_length = 0
+          end
+          current_length += 1  # this is for newline
+        end
+        if current_message.length > 0
+          messages_to_send.push(current_message)
+        end
+      end
+      
+      messages_to_send.each_with_index do |message, i|
+        retries = 0
+        start_time = Time.now
+        sleep_time = @retry_min_interval
+
+        while true
+          common_log_part = "#{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}', chunk #{chunk_id}, try #{retries}, batch #{i}"
+
+          begin
+            @log.debug { "Sending #{message.count}; #{common_log_part}" }
+
+            @sumo_conn.publish(
+              message.join("\n"),
+                source_host         =source_host,
+                source_category     =source_category,
+                source_name         =source_name,
+                data_type           =@data_type,
+                metric_data_format  =@metric_data_format,
+                collected_fields    =fields,
+                dimensions          =@custom_dimensions
+            )
+            break
+          rescue => e
+            if !@use_internal_retry
+              raise e
+            end
+            # increment retries
+            retries += 1
+
+            log.warn "error while sending request to sumo: #{e}; #{common_log_part}"
+            log.warn_backtrace e.backtrace
+
+            # drop data if
+            #   - we reached out the @retry_max_times retries
+            #   - or we exceeded @retry_timeout
+            if (retries >= @retry_max_times && @retry_max_times > 0) || (Time.now > start_time + @retry_timeout && @retry_timeout > 0)
+              log.warn "dropping records; #{common_log_part}"
+              break
+            end
+
+            log.info "going to retry to send data at #{Time.now + sleep_time}; #{common_log_part}"
+            sleep sleep_time
+
+            sleep_time *= 2
+            if sleep_time > @retry_max_interval
+              sleep_time = @retry_max_interval
+            end
+          end
+        end
+      end
     end
 
   end

data/test/plugin/test_out_sumologic.rb

@@ -825,4 +825,157 @@ class SumologicOutput < Test::Unit::TestCase
     end
   end
 
+  def test_resend
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 3
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      {status: 500, headers: {content_type: 'application/json'}},
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:2
+  end
+
+  def test_resend_failed
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 15
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      status: 500, headers: {content_type: 'application/json'}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:15
+  end
+
+  def test_resend_forever
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 0
+      retry_timeout 0s
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      *[{status: 500, headers: {content_type: 'application/json'}}]*123,
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:124
+  end
+
+  def test_skip_retry
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      endpoint #{endpoint}
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 500, headers: {content_type: 'application/json'})
+
+    exception = assert_raise(RuntimeError) {
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+    }
+    assert_equal("Failed to send data to HTTP Source. 500 - ", exception.message)
+  end
+
+  def test_split_negative_or_zero
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    configs = [
+      %{
+        endpoint #{endpoint}
+        max_request_size -5
+      },
+      %{
+        endpoint #{endpoint}
+        max_request_size 0
+      }
+    ]
+
+    time = event_time
+
+    configs.each do |config|
+      WebMock.reset_executed_requests!
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+      end
+
+      assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      body: /\A{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\z/,
+                      times:1
+    end
+  end
+
+  def test_split
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    config = %{
+        endpoint #{endpoint}
+        max_request_size 80
+      }
+
+    time = event_time
+
+    WebMock.reset_executed_requests!
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+    driver.run do
+      driver.feed("test", time, {"message": "test1"})
+      driver.feed("test", time, {"message": "test2"})
+      driver.feed("test", time, {"message": "test3"})
+      driver.feed("test", time, {"message": "test4"})
+      driver.feed("test", time, {"message": "test5"})
+    end
+
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test1"}\n{"timestamp":\d+.,"message":"test2"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test3"}\n{"timestamp":\d+.,"message":"test4"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test5"}\z/,
+                    times:1
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.5
+  version: 1.8.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-11 00:00:00.000000000 Z
+date: 2022-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler