fluent-plugin-sumologic_output 1.7.5 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10221368588a63fe0f97cb60695466a78a12eae80c13f736b5ae54879e4a5b4e
-  data.tar.gz: 2b99181908e9df7b2bcae1e6534a9311d72464cb74923a77f3a7489eeeb13f94
+  metadata.gz: cbb31ea7d5a93f777e1c509790dec08a6462bade567f4ff222a62c628e3790b4
+  data.tar.gz: ed18ad85714d06efaee5c68689dfc36e4e78794e3a326602f46fd5947c28ee6a
 SHA512:
-  metadata.gz: 245aa88c7c415099db251eabf2c6b9e80a64d657cdf4df90e365115e962cbeec76279cab1d4d5dc87de1b6387475c0e6363a2be24c55d21953d4bc852727d869
-  data.tar.gz: c9a870b1252ca73b2565a4627e29ee5501ce9da6caa88834932d0edee05e45f573a5f7aeae183968e303786615622487188287236b1d12d8234ac992f25e822a
+  metadata.gz: 20ee2dc1b70ea6845e8251711e997ce08c682c2c5bdbf923cbef8b26a907498174a8ea77dcac8f9e405bead5deaa2acff1cc560c9c2c9dfa41c18f1c549372b1
+  data.tar.gz: 9c6bbd8fb280eba89718191a68f6d79a1f244f34d863aa9aa621e47766ad20769e98a5ee6718a77b2934551562b9c8603b06879de30f51fd3f9d24a3e6252dca

data/.github/workflows/publish.yaml

@@ -0,0 +1,23 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+'
+
+jobs:
+  publish:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build
+        run: gem build fluent-plugin-sumologic_output.gemspec
+      - name: Publish
+        env:
+          GEM_HOST_API_KEY: ${{ secrets.RUGYGEMS_APIKEY }}
+        run: gem push fluent-plugin-sumologic_output-*.gem

data/.github/workflows/pull_request.yaml

@@ -0,0 +1,15 @@
+name: PR check
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+      - run: bundle install
+      - run: bundle exec rake

data/CHANGELOG.md

@@ -2,6 +2,35 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+## [1.9.0]
+
+Released 2024-02-14
+
+- feat: enable compression by default [#87](https://github.com/SumoLogic/fluentd-output-sumologic/pull/87)
+- feat: log warning if `log_key` does not exist in log [#86](https://github.com/SumoLogic/fluentd-output-sumologic/pull/86)
+- fix: fix how `compress` configuration flag works [#90](https://github.com/SumoLogic/fluentd-output-sumologic/pull/90)
+
+  In `v1.8.0`, setting `compress` flag to either `true` or `false` caused compression to be enabled.
+  This is now fixed.
+
+[1.9.0]: https://github.com/SumoLogic/fluentd-output-sumologic/releases/1.9.0
+
+<a name="1.8.0"></a>
+# [1.8.0] (2022-04-22)
+
+- feat: add exponential backoff for sending data [#76](https://github.com/SumoLogic/fluentd-output-sumologic/pull/76)
+- feat(max_request_size): add max_request_size to limit size of requests [#78](https://github.com/SumoLogic/fluentd-output-sumologic/pull/78)
+
+<a name="1.7.5"></a>
+# [1.7.5] (2022-04-11)
+
+- refactor: add a debug log on sending [#75](https://github.com/SumoLogic/fluentd-output-sumologic/pull/75)
+
+<a name="1.7.4"></a>
+# [1.7.4] (2021-04-08)
+
+- fix: handle receiver warning messages [#73](https://github.com/SumoLogic/fluentd-output-sumologic/pull/73)
+
 <a name="1.7.3"></a>
 # [1.7.3] (2021-10-19)
 - Expose httpclient send_timeout [#66](https://github.com/SumoLogic/fluentd-output-sumologic/pull/68)

data/CONTRIBUTING.md

@@ -0,0 +1,23 @@
+# Contributing
+
+## How to release
+
+1. Create a pull request named `chore: release x.y.z` with the following changes:
+
+   - Set `gem.version` to `"x.y.z"` in [fluent-plugin-sumologic_output.gemspec](fluent-plugin-sumologic_output.gemspec).
+   - Add new version to [CHANGELOG.md](./CHANGELOG.md).
+
+2. Create and push the release tag:
+
+   ```bash
+   git checkout main
+   git pull
+   export VERSION=x.y.z
+   git tag -a "v${VERSION}" -m "Release v${VERSION}"
+   git push origin "v${VERSION}"
+   ```
+
+   This will trigger the GitHub Actions [publish](./.github/workflows/publish.yaml) action to pubilsh the gem in Ruby Gems.
+
+3. Go to https://github.com/SumoLogic/fluentd-output-sumologic/releases and create a new release for the tag.
+   Copy the changes from Changelog and publish the release.

data/README.md

@@ -34,10 +34,17 @@ Configuration options for fluent.conf are:
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
 * `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
 * `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
-* `compress` - Option to enable compression (default `false`)
+* `compress` - Option to enable compression (default `true`)
 * `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
 * `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
 * `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
+* `use_internal_retry` - Enable custom retry mechanism. As this is `false` by default due to backward compatibility,
+  we recommend to enable it and configure the following parameters (`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times`)
+* `retry_min_interval` - Minimum interval to wait between sending tries (default is `1s`)
+* `retry_max_interval` - Maximum interval to wait between sending tries (default is `5m`)
+* `retry_timeout` - Time after which the data is going to be dropped (default is `72h`) (`0s` means that there is no timeout)
+* `retry_max_times` - Maximum number of retries (default is `0`) (`0` means that there is no max retry times, retries will happen forever)
+* `max_request_size` - Maximum request size (before applying compression). Default is `0k` which means no limit
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 
@@ -137,6 +144,14 @@ Example
 }
 ```
 
+## Retry Mechanism
+
+`retry_min_interval`, `retry_max_interval`, `retry_timeout`, `retry_max_times` are not the [buffer retries parameters][buffer_retries].
+Due to technical reason, this plugin implements it's own retrying back-off exponential mechanism.
+It is disabled by default, but we recommend to enable it by setting `use_internal_retry` to `true`.
+
+[buffer_retries]: https://docs.fluentd.org/configuration/buffer-section#retries-parameters
+
 ### TLS 1.2 Requirement
 
 Sumo Logic only accepts connections from clients using TLS version 1.2 or greater. To utilize the content of this repo, ensure that it's running in an execution environment that is configured to use TLS 1.2 or greater.

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.7.5"
+  gem.version       = "1.9.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}
@@ -17,12 +17,10 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 2.0.0'
-
-  gem.add_development_dependency "bundler", "~> 2"
+  gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
-  gem.add_development_dependency 'test-unit', '~> 3.1.0'
-  gem.add_development_dependency "codecov", ">= 0.1.10"
-  gem.add_runtime_dependency "fluentd", ">= 0.14.12"
-  gem.add_runtime_dependency 'httpclient', '~> 2.8.0'
+  gem.add_development_dependency 'test-unit'
+  gem.add_development_dependency "codecov"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency 'httpclient'
 end

data/lib/fluent/plugin/out_sumologic.rb

@@ -157,13 +157,22 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+
+  config_param :use_internal_retry, :bool, :default => false
+  config_param :retry_timeout, :time, :default => 72 * 3600  # 72h
+  config_param :retry_max_times, :integer, :default => 0
+  config_param :retry_min_interval, :time, :default => 1  # 1s
+  config_param :retry_max_interval, :time, :default => 5*60  # 5m
+
+  config_param :max_request_size, :size, :default => 0
+
   # https://help.sumologic.com/Manage/Fields
   desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
   config_param :custom_fields, :string, :default => nil
   desc 'Name of sumo client which is send as X-Sumo-Client header'
   config_param :sumo_client, :string, :default => 'fluentd-output'
   desc 'Compress payload'
-  config_param :compress, :bool, :default => false
+  config_param :compress, :bool, :default => true
   desc 'Encoding method of compresssion (either gzip or deflate)'
   config_param :compress_encoding, :string, :default => SumologicConnection::COMPRESS_GZIP
   # https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers
@@ -187,65 +196,50 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   def configure(conf)
 
     compat_parameters_convert(conf, :buffer)
+    super
 
-    unless conf['endpoint'] =~ URI::regexp
-      raise Fluent::ConfigError, "Invalid SumoLogic endpoint url: #{conf['endpoint']}"
+    unless @endpoint =~ URI::regexp
+      raise Fluent::ConfigError, "Invalid SumoLogic endpoint url: #{@endpoint}"
     end
 
-    unless conf['data_type'].nil?
-      unless conf['data_type'] =~ /\A(?:logs|metrics)\z/
-        raise Fluent::ConfigError, "Invalid data_type #{conf['data_type']} must be logs or metrics"
-      end
+    unless @data_type =~ /\A(?:logs|metrics)\z/
+      raise Fluent::ConfigError, "Invalid data_type #{@data_type} must be logs or metrics"
     end
 
-    if conf['data_type'].nil? || conf['data_type'] == LOGS_DATA_TYPE
-      unless conf['log_format'].nil?
-        unless conf['log_format'] =~ /\A(?:json|text|json_merge|fields)\z/
-          raise Fluent::ConfigError, "Invalid log_format #{conf['log_format']} must be text, json, json_merge or fields"
-        end
+    if @data_type == LOGS_DATA_TYPE
+      unless @log_format =~ /\A(?:json|text|json_merge|fields)\z/
+        raise Fluent::ConfigError, "Invalid log_format #{@log_format} must be text, json, json_merge or fields"
       end
     end
 
-    if conf['data_type'] == METRICS_DATA_TYPE && ! conf['metrics_data_type'].nil?
-      unless conf['metrics_data_type'] =~ /\A(?:graphite|carbon2|pronetheus)\z/
-        raise Fluent::ConfigError, "Invalid metrics_data_type #{conf['metrics_data_type']} must be graphite or carbon2 or prometheus"
+    if @data_type == METRICS_DATA_TYPE
+      unless @metric_data_format =~ /\A(?:graphite|carbon2|prometheus)\z/
+        raise Fluent::ConfigError, "Invalid metric_data_format #{@metric_data_format} must be graphite or carbon2 or prometheus"
       end
     end
 
-    conf['custom_fields'] = validate_key_value_pairs(conf['custom_fields'])
-    if conf['custom_fields'].nil?
-      conf.delete 'custom_fields'
-    end
-    unless conf['custom_fields']
-      @log.debug "Custom fields: #{conf['custom_fields']}"
+    @custom_fields = validate_key_value_pairs(@custom_fields)
+    if @custom_fields
+      @log.debug "Custom fields: #{@custom_fields}"
     end
 
-    conf['custom_dimensions'] = validate_key_value_pairs(conf['custom_dimensions'])
-    if conf['custom_dimensions'].nil?
-      conf.delete 'custom_dimensions'
-    end
-    unless conf['custom_dimensions']
-      @log.debug "Custom dimensions: #{conf['custom_dimensions']}"
-    end
-
-    # For some reason default is set incorrectly in unit-tests
-    if conf['sumo_client'].nil? || conf['sumo_client'].strip.length == 0
-      conf['sumo_client'] = 'fluentd-output'
+    @custom_dimensions = validate_key_value_pairs(@custom_dimensions)
+    if @custom_dimensions
+      @log.debug "Custom dimensions: #{@custom_dimensions}"
     end
 
     @sumo_conn = SumologicConnection.new(
-      conf['endpoint'],
-      conf['verify_ssl'],
-      conf['open_timeout'].to_i,
-      conf['send_timeout'].to_i,
-      conf['proxy_uri'],
-      conf['disable_cookies'],
-      conf['sumo_client'],
-      conf['compress'],
-      conf['compress_encoding'],
-      log,
+      @endpoint,
+      @verify_ssl,
+      @open_timeout,
+      @send_timeout,
+      @proxy_uri,
+      @disable_cookies,
+      @sumo_client,
+      @compress,
+      @compress_encoding,
+      @log,
       )
-    super
   end
 
   # This method is called when starting.
@@ -354,6 +348,9 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       when 'logs'
         case log_format
         when 'text'
+          if !record.has_key?(@log_key)
+            log.warn "log key `#{@log_key}` has not been found in the log"
+          end
           log = log_to_str(record[@log_key])
         when 'json_merge'
           if @add_timestamp
@@ -385,6 +382,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     end
 
+    chunk_id = "##{chunk.dump_unique_id_hex(chunk.unique_id)}"
     # Push logs to sumo
     messages_list.each do |key, messages|
       source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
@@ -397,18 +395,78 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
         fields = [fields,@custom_fields].compact.join(",")
       end
 
-      @log.debug { "Sending #{messages.count} #{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}'." }
-
-      @sumo_conn.publish(
-          messages.join("\n"),
-          source_host         =source_host,
-          source_category     =source_category,
-          source_name         =source_name,
-          data_type           =@data_type,
-          metric_data_format  =@metric_data_format,
-          collected_fields    =fields,
-          dimensions          =@custom_dimensions
-      )
+      if @max_request_size <= 0
+        messages_to_send = [messages]
+      else
+        messages_to_send = []
+        current_message = []
+        current_length = 0
+        messages.each do |message|
+          current_message.push message
+          current_length += message.length
+
+          if current_length > @max_request_size
+            messages_to_send.push(current_message)
+            current_message = []
+            current_length = 0
+          end
+          current_length += 1  # this is for newline
+        end
+        if current_message.length > 0
+          messages_to_send.push(current_message)
+        end
+      end
+      
+      messages_to_send.each_with_index do |message, i|
+        retries = 0
+        start_time = Time.now
+        sleep_time = @retry_min_interval
+
+        while true
+          common_log_part = "#{@data_type} records with source category '#{source_category}', source host '#{source_host}', source name '#{source_name}', chunk #{chunk_id}, try #{retries}, batch #{i}"
+
+          begin
+            @log.debug { "Sending #{message.count}; #{common_log_part}" }
+
+            @sumo_conn.publish(
+              message.join("\n"),
+                source_host         =source_host,
+                source_category     =source_category,
+                source_name         =source_name,
+                data_type           =@data_type,
+                metric_data_format  =@metric_data_format,
+                collected_fields    =fields,
+                dimensions          =@custom_dimensions
+            )
+            break
+          rescue => e
+            if !@use_internal_retry
+              raise e
+            end
+            # increment retries
+            retries += 1
+
+            log.warn "error while sending request to sumo: #{e}; #{common_log_part}"
+            log.warn_backtrace e.backtrace
+
+            # drop data if
+            #   - we reached out the @retry_max_times retries
+            #   - or we exceeded @retry_timeout
+            if (retries >= @retry_max_times && @retry_max_times > 0) || (Time.now > start_time + @retry_timeout && @retry_timeout > 0)
+              log.warn "dropping records; #{common_log_part}"
+              break
+            end
+
+            log.info "going to retry to send data at #{Time.now + sleep_time}; #{common_log_part}"
+            sleep sleep_time
+
+            sleep_time *= 2
+            if sleep_time > @retry_max_interval
+              sleep_time = @retry_max_interval
+            end
+          end
+        end
+      end
     end
 
   end

data/test/plugin/test_out_sumologic.rb

@@ -19,7 +19,15 @@ class SumologicOutput < Test::Unit::TestCase
   def test_no_endpoint_configure
     config = %{}
     exception = assert_raise(Fluent::ConfigError) {create_driver(config)}
-    assert_equal("Invalid SumoLogic endpoint url: ", exception.message)
+    assert_equal("'endpoint' parameter is required", exception.message)
+  end
+
+  def test_invalid_endpoint
+    config = %{
+      endpoint Not-a-URL
+    }
+    exception = assert_raise(Fluent::ConfigError) {create_driver(config)}
+    assert_equal("Invalid SumoLogic endpoint url: Not-a-URL", exception.message)
   end
 
   def test_invalid_data_type_configure
@@ -40,14 +48,14 @@ class SumologicOutput < Test::Unit::TestCase
     assert_equal("Invalid log_format foo must be text, json, json_merge or fields", exception.message)
   end
 
-  def test_invalid_metrics_data_type
+  def test_invalid_metric_data_format
     config = %{
       endpoint            https://SUMOLOGIC_URL
       data_type           metrics
-      metrics_data_type   foo
+      metric_data_format  foo
     }
     exception = assert_raise(Fluent::ConfigError) {create_driver(config)}
-    assert_equal("Invalid metrics_data_type foo must be graphite or carbon2 or prometheus", exception.message)
+    assert_equal("Invalid metric_data_format foo must be graphite or carbon2 or prometheus", exception.message)
   end
 
   def test_default_configure
@@ -73,11 +81,13 @@ class SumologicOutput < Test::Unit::TestCase
     assert_equal instance.proxy_uri, nil
     assert_equal instance.disable_cookies, false
     assert_equal instance.sumo_client, 'fluentd-output'
+    assert_equal instance.compress, true
     assert_equal instance.compress_encoding, 'gzip'
   end
 
   def test_emit_text
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      text
       source_category test
@@ -99,6 +109,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_text_custom_sumo_client
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      text
       source_category test
@@ -121,6 +132,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
       source_category test
@@ -142,6 +154,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_empty_fields
     config = %{
+        compress        false
 	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
 	      log_format      fields
 	      source_category test
@@ -163,6 +176,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json_double_encoded
     config = %{
+      compress        false
       endpoint        https://endpoint3.collection.us2.sumologic.com/receiver/v1/http/1234
       log_format      json
       source_category test
@@ -184,6 +198,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_text_format_as_json
     config = %{
+      compress        false
       endpoint        https://endpoint3.collection.us2.sumologic.com/receiver/v1/http/1234
       log_format      json
       source_category test
@@ -205,6 +220,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json_merge
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json_merge
       source_category test
@@ -226,6 +242,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json_merge_timestamp
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json_merge
       source_category test
@@ -247,6 +264,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata_with_fields_json_format
     config = %{
+      compress        false
 	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
 	      log_format      json
 	    }
@@ -270,6 +288,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata_with_fields_fields_format
     config = %{
+        compress        false
 	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
 	      log_format      fields
 	    }
@@ -293,6 +312,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata_with_fields_and_custom_fields_fields_format
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       custom_fields   "lorem=ipsum,dolor=amet"
@@ -317,6 +337,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata_with_fields_and_empty_custom_fields_fields_format
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       custom_fields   ""
@@ -341,6 +362,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata_with_empty_fields_and_custom_fields_fields_format
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       custom_fields   "lorem=ipsum,invalid"
@@ -365,6 +387,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_with_sumo_metadata
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
     }
@@ -387,6 +410,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json_no_timestamp
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
       source_category test
@@ -408,6 +432,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_json_timestamp_key
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
       source_category test
@@ -429,6 +454,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_graphite
     config = %{
+      compress            false
       endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
       data_type           metrics
       metric_data_format  graphite
@@ -450,6 +476,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_carbon
     config = %{
+      compress            false
       endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
       data_type           metrics
       metric_data_format  carbon2
@@ -471,6 +498,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_prometheus
     config = %{
+      compress            false
       endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
       data_type           metrics
       metric_data_format  prometheus
@@ -492,6 +520,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_prometheus_with_custom_dimensions
     config = %{
+      compress            false
       endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
       data_type           metrics
       metric_data_format  prometheus
@@ -520,6 +549,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_prometheus_with_empty_custom_metadata
     config = %{
+      compress            false
       endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
       data_type           metrics
       metric_data_format  prometheus
@@ -547,6 +577,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_batching_same_headers
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
       source_category test
@@ -568,6 +599,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_batching_different_headers
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      json
       source_category test
@@ -593,6 +625,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_batching_different_fields
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       source_category test
@@ -635,7 +668,6 @@ class SumologicOutput < Test::Unit::TestCase
       source_name       test
       compress          true
       compress_encoding deflate
-
     }
     driver = create_driver(config)
     time = event_time
@@ -657,7 +689,6 @@ class SumologicOutput < Test::Unit::TestCase
       source_host       test
       source_name       test
       compress          true
-
     }
     driver = create_driver(config)
     time = event_time
@@ -673,6 +704,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_text_from_array
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      text
       source_category test
@@ -694,6 +726,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_text_from_dict
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      text
       source_category test
@@ -715,6 +748,7 @@ class SumologicOutput < Test::Unit::TestCase
 
   def test_emit_fields_string_based
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       source_category test
@@ -736,6 +770,7 @@ class SumologicOutput < Test::Unit::TestCase
   
   def test_emit_fields_invalid_json_string_based_1
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       source_category test
@@ -757,6 +792,7 @@ class SumologicOutput < Test::Unit::TestCase
   
   def test_emit_fields_invalid_json_string_based_2
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       source_category test
@@ -778,6 +814,7 @@ class SumologicOutput < Test::Unit::TestCase
   
   def test_emit_fields_invalid_json_string_based_3
     config = %{
+      compress        false
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
       log_format      fields
       source_category test
@@ -800,6 +837,7 @@ class SumologicOutput < Test::Unit::TestCase
   def test_warning_response_from_receiver
     endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
     config = %{
+      compress false
       endpoint #{endpoint}
     }
     testdata = [
@@ -825,4 +863,164 @@ class SumologicOutput < Test::Unit::TestCase
     end
   end
 
+  def test_resend
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      compress false
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 3
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      {status: 500, headers: {content_type: 'application/json'}},
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:2
+  end
+
+  def test_resend_failed
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      compress false
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 15
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      status: 500, headers: {content_type: 'application/json'}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:15
+  end
+
+  def test_resend_forever
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      compress false
+      endpoint #{endpoint}
+      retry_min_interval 0s
+      retry_max_times 0
+      retry_timeout 0s
+      use_internal_retry true
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(
+      *[{status: 500, headers: {content_type: 'application/json'}}]*123,
+      {status: 200, headers: {content_type: 'application/json'}}
+    )
+    driver.run do
+      driver.feed("test", time, {"message": "test"})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:124
+  end
+
+  def test_skip_retry
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+    config = %{
+      compress false
+      endpoint #{endpoint}
+    }
+    time = event_time
+
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 500, headers: {content_type: 'application/json'})
+
+    exception = assert_raise(RuntimeError) {
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+      end
+    }
+    assert_equal("Failed to send data to HTTP Source. 500 - ", exception.message)
+  end
+
+  def test_split_negative_or_zero
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    configs = [
+      %{
+        compress false
+        endpoint #{endpoint}
+        max_request_size -5
+      },
+      %{
+        compress false
+        endpoint #{endpoint}
+        max_request_size 0
+      }
+    ]
+
+    time = event_time
+
+    configs.each do |config|
+      WebMock.reset_executed_requests!
+      driver = create_driver(config)
+      stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+      driver.run do
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+        driver.feed("test", time, {"message": "test"})
+      end
+
+      assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      body: /\A{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\n{"timestamp":\d+.,"message":"test"}\z/,
+                      times:1
+    end
+  end
+
+  def test_split
+    endpoint = "https://collectors.sumologic.com/v1/receivers/http/1234"
+
+    config = %{
+        compress false
+        endpoint #{endpoint}
+        max_request_size 80
+      }
+
+    time = event_time
+
+    WebMock.reset_executed_requests!
+    driver = create_driver(config)
+    stub_request(:post, endpoint).to_return(status: 200, headers: {content_type: 'application/json'})
+
+    driver.run do
+      driver.feed("test", time, {"message": "test1"})
+      driver.feed("test", time, {"message": "test2"})
+      driver.feed("test", time, {"message": "test3"})
+      driver.feed("test", time, {"message": "test4"})
+      driver.feed("test", time, {"message": "test5"})
+    end
+
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test1"}\n{"timestamp":\d+.,"message":"test2"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test3"}\n{"timestamp":\d+.,"message":"test4"}\z/,
+                    times:1
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                    body: /\A{"timestamp":\d+.,"message":"test5"}\z/,
+                    times:1
+  end
+
 end

data/vagrant/provision.sh

@@ -22,6 +22,5 @@ usermod -aG docker vagrant
 apt-get install -y make
 
 # install requirements for ruby
-snap install ruby --channel=2.6/stable --classic
-su vagrant -c 'gem install bundler:2.1.4'
+snap install ruby --channel=3.3/stable --classic
 apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.7.5
+  version: 1.9.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-11 00:00:00.000000000 Z
+date: 2024-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -43,58 +43,58 @@ dependencies:
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.12
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.12
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: '0'
 description: Output plugin to SumoLogic HTTP Endpoint
 email:
 - stevezau@gmail.com
@@ -104,9 +104,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
+- ".github/workflows/publish.yaml"
+- ".github/workflows/pull_request.yaml"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
+- CONTRIBUTING.md
 - Gemfile
 - LICENSE
 - README.md
@@ -131,14 +134,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint