fluent-plugin-sumologic_output 1.5.0 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6db47bbad47c3eb7ea578b1e894ddc972336e785c94bb8af47595481b7e934e2
-  data.tar.gz: 3457257200adc1b655ed1820a5041e29c22531e621aa24a6dd620105ac4559f1
+  metadata.gz: c3bf755e8ecd488457805d948614c43b3657de815889dd341091d11f338ad995
+  data.tar.gz: 20163f7c5ac543a010e53fce506dbaed10bafe5d46830579a3101558f5b61fa4
 SHA512:
-  metadata.gz: cad8b409225fdd0d96402c17a950c0fb67ad61d0f28ac98a252bac23671cee9a3152738c37f916295c116c0396ede9ea874339c8d4278daf81f698060b56f63d
-  data.tar.gz: 619b7cc028a11d1485b158a08e5f34f00440e5410b14b4283d0dda778f80d22944be9f753275a3028c360e15a69baf437b761a09c7e6b29e2f2c784bf3c0d965
+  metadata.gz: 35cf3af9a561fd22e810dea880570007613b7cb9ae87307dd46ea3ff8d9e005b96c9ebf19f1aafd58b6ae4495509f51406b7788029f7fb26144f54b60f22370a
+  data.tar.gz: 6b5e6a13ed50d798c4c0f8225e771a2c5977373373aed316cbe1a5369a2af22aef56c86e474aefd6ca847e41c496c287a191c23c1224d2a49075794b420949e4

data/CHANGELOG.md

@@ -2,7 +2,21 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
-<a name="1.4.0"></a>
+<a name="1.7.1"></a>
+# [1.7.1] (2020-04-28)
+- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
+
+<a name="1.7.0"></a>
+# [1.7.0] (2020-04-23)
+- Add option for specifing custom fields for logs: [#56](https://github.com/SumoLogic/fluentd-output-sumologic/pull/56)
+- Add option for specifing custom dimensions for metrics: [#57](https://github.com/SumoLogic/fluentd-output-sumologic/pull/57)
+- Add support for compression: [#58](https://github.com/SumoLogic/fluentd-output-sumologic/pull/58)
+
+<a name="1.5.0"></a>
+# [1.5.0] (2019-06-26)
+- Add support for new log format fields: [#49](https://github.com/SumoLogic/fluentd-output-sumologic/pull/49)
+
+<a name="1.4.1"></a>
 # [1.4.1] (2019-03-13)
 
 - Add option for sending metrics in Prometheus format [#39](https://github.com/SumoLogic/fluentd-output-sumologic/pull/39) 

data/README.md

@@ -4,9 +4,6 @@
 
 This plugin has been designed to output logs or metrics to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source)
 
-## Support
-The code in this repository has been developed in collaboration with the Sumo Logic community and is not supported via standard Sumo Logic Support channels. For any issues or questions please submit an issue within the GitHub repository. The maintainers of this project will work directly with the community to answer any questions, address bugs, or review any requests for new features. 
-
 ## License
 Released under Apache 2.0 License. 
 
@@ -36,6 +33,10 @@ Configuration options for fluent.conf are:
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
 * `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
 * `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
+* `compress` - Option to enable compression (default `false`)
+* `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
+* `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
+* `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.5.0"
+  gem.version       = "1.7.2"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -2,30 +2,47 @@ require 'fluent/plugin/output'
 require 'net/https'
 require 'yajl'
 require 'httpclient'
+require 'zlib'
+require 'stringio'
 
 class SumologicConnection
 
   attr_reader :http
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  COMPRESS_DEFLATE = 'deflate'
+  COMPRESS_GZIP = 'gzip'
+
+  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
     @endpoint = endpoint
+    @sumo_client = sumo_client
     create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    @compress = compress_enabled
+    @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+
+    unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
+      raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
+    end
   end
 
-  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields)
-    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields))
+  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields, dimensions)
+    response = http.post(@endpoint, compress(raw_data), request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields, dimensions))
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
   end
 
-  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields)
+  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
     headers = {
         'X-Sumo-Name'     => source_name,
         'X-Sumo-Category' => source_category,
         'X-Sumo-Host'     => source_host,
-        'X-Sumo-Client'   => 'fluentd-output'
+        'X-Sumo-Client'   => @sumo_client,
     }
+
+    if @compress
+      headers['Content-Encoding'] = @compress_encoding
+    end
+
     if data_type == 'metrics'
       case metric_data_format
       when 'graphite'
@@ -37,6 +54,10 @@ class SumologicConnection
       else
         raise RuntimeError, "Invalid #{metric_data_format}, must be graphite or carbon2 or prometheus"
       end
+
+      unless dimensions.nil?
+        headers['X-Sumo-Dimensions'] = dimensions
+      end
     end
     unless collected_fields.nil?
       headers['X-Sumo-Fields'] = collected_fields
@@ -56,6 +77,29 @@ class SumologicConnection
       @http.cookie_manager       = nil
     end
   end
+
+  def compress(content)
+    if @compress
+      if @compress_encoding == COMPRESS_GZIP
+        result = gzip(content)
+        result.bytes.to_a.pack("c*")
+      else
+        Zlib::Deflate.deflate(content)
+      end
+    else
+      content
+    end
+  end # def compress
+  
+  def gzip(content)
+    stream = StringIO.new("w")
+    stream.set_encoding("ASCII")
+    gz = Zlib::GzipWriter.new(stream)
+    gz.mtime=1  # Ensure that for same content there is same output
+    gz.write(content)
+    gz.close
+    stream.string.bytes.to_a.pack("c*")
+  end # def gzip
 end
 
 class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
@@ -86,6 +130,18 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+  # https://help.sumologic.com/Manage/Fields
+  desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
+  config_param :custom_fields, :string, :default => nil
+  desc 'Name of sumo client which is send as X-Sumo-Client header'
+  config_param :sumo_client, :string, :default => 'fluentd-output'
+  desc 'Compress payload'
+  config_param :compress, :bool, :default => false
+  desc 'Encoding method of compresssion (either gzip or deflate)'
+  config_param :compress_encoding, :string, :default => SumologicConnection::COMPRESS_GZIP
+  # https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers
+  desc 'Dimensions string (eg "cluster=payment, service=credit_card") which is going to be added to every metric record.'
+  config_param :custom_dimensions, :string, :default => nil
 
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -129,7 +185,37 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       end
     end
 
-    @sumo_conn = SumologicConnection.new(conf['endpoint'], conf['verify_ssl'], conf['open_timeout'].to_i, conf['proxy_uri'], conf['disable_cookies'])
+    conf['custom_fields'] = validate_key_value_pairs(conf['custom_fields'])
+    if conf['custom_fields'].nil?
+      conf.delete 'custom_fields'
+    end
+    unless conf['custom_fields']
+      @log.debug "Custom fields: #{conf['custom_fields']}"
+    end
+
+    conf['custom_dimensions'] = validate_key_value_pairs(conf['custom_dimensions'])
+    if conf['custom_dimensions'].nil?
+      conf.delete 'custom_dimensions'
+    end
+    unless conf['custom_dimensions']
+      @log.debug "Custom dimensions: #{conf['custom_dimensions']}"
+    end
+
+    # For some reason default is set incorrectly in unit-tests
+    if conf['sumo_client'].nil? || conf['sumo_client'].strip.length == 0
+      conf['sumo_client'] = 'fluentd-output'
+    end
+
+    @sumo_conn = SumologicConnection.new(
+      conf['endpoint'],
+      conf['verify_ssl'],
+      conf['open_timeout'].to_i,
+      conf['proxy_uri'],
+      conf['disable_cookies'],
+      conf['sumo_client'],
+      conf['compress'],
+      conf['compress_encoding']
+      )
     super
   end
 
@@ -195,7 +281,11 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     source_host = sumo_metadata['host'] || @source_host
     source_host = extract_placeholders(source_host, chunk) unless source_host.nil?
 
-    "#{source_name}:#{source_category}:#{source_host}"
+    fields = sumo_metadata['fields'] || ""
+    fields = extract_placeholders(fields, chunk) unless fields.nil?
+
+    { :source_name => "#{source_name}", :source_category => "#{source_category}",
+      :source_host => "#{source_host}", :fields => "#{fields}" }
   end
 
   # Convert timestamp to 13 digit epoch if necessary
@@ -203,28 +293,22 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     time.to_s.length == 13 ? time : time * 1000
   end
 
-  def sumo_fields(sumo_metadata)
-    fields = sumo_metadata['fields'] || ""
-    Hash[
-        fields.split(',').map do |pair|
-          k, v = pair.split('=', 2)
-          [k, v]
-        end
-    ]
-  end
+  # Convert log to string and strip it
+  def log_to_str(log)
+    if log.is_a?(Array) or log.is_a?(Hash)
+      log = Yajl.dump(log)
+    end
 
-  def dump_collected_fields(log_fields)
-    if log_fields.nil?
-      log_fields
-    else
-      log_fields.map{|k,v| "#{k}=#{v}"}.join(',')
+    unless log.nil?
+      log.strip!
     end
+
+    return log
   end
 
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
     messages_list = {}
-    log_fields = nil
 
     # Sort messages
     chunk.msgpack_each do |time, record|
@@ -242,17 +326,13 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       when 'logs'
         case log_format
         when 'text'
-          log = record[@log_key]
-          unless log.nil?
-            log.strip!
-          end
+          log = log_to_str(record[@log_key])
         when 'json_merge'
           if @add_timestamp
             record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
           end
           log = dump_log(merge_json(record))
         when 'fields'
-          log_fields = sumo_fields(sumo_metadata)
           if @add_timestamp
             record = {  @timestamp_key => sumo_timestamp(time) }.merge(record)
           end
@@ -264,10 +344,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           log = dump_log(record)
         end
       when 'metrics'
-        log = record[@log_key]
-        unless log.nil?
-          log.strip!
-        end
+        log = log_to_str(record[@log_key])
       end
 
       unless log.nil?
@@ -282,7 +359,16 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     # Push logs to sumo
     messages_list.each do |key, messages|
-      source_name, source_category, source_host = key.split(':')
+      source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
+        key[:source_host], key[:fields]
+
+      # Merge custom and record fields
+      if fields.nil? || fields.strip.length == 0
+        fields = @custom_fields
+      else
+        fields = [fields,@custom_fields].compact.join(",")
+      end
+
       @sumo_conn.publish(
           messages.join("\n"),
           source_host         =source_host,
@@ -290,9 +376,26 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           source_name         =source_name,
           data_type           =@data_type,
           metric_data_format  =@metric_data_format,
-          collected_fields    =dump_collected_fields(log_fields)
+          collected_fields    =fields,
+          dimensions          =@custom_dimensions
       )
     end
 
   end
+
+  def validate_key_value_pairs(fields)
+    if fields.nil?
+      return fields
+    end
+
+    fields = fields.split(",").select { |field|
+      field.split('=').length == 2
+    }
+
+    if fields.length == 0
+      return nil
+    end
+
+    fields.join(',')
+  end
 end

data/test/plugin/test_out_sumologic.rb

@@ -72,6 +72,8 @@ class SumologicOutput < Test::Unit::TestCase
     assert_equal instance.timestamp_key, 'timestamp'
     assert_equal instance.proxy_uri, nil
     assert_equal instance.disable_cookies, false
+    assert_equal instance.sumo_client, 'fluentd-output'
+    assert_equal instance.compress_encoding, 'gzip'
   end
 
   def test_emit_text
@@ -95,6 +97,28 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_text_custom_sumo_client
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+      sumo_client     'fluentd-custom-sender'
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => 'test'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-custom-sender', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: "test",
+                     times:1
+  end
+
   def test_emit_json
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -267,6 +291,78 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_with_sumo_metadata_with_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,dolor=amet"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic,lorem=ipsum,dolor=amet'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_fields_and_empty_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   ""
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_empty_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,invalid"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": ""
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'lorem=ipsum'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
   def test_emit_with_sumo_metadata
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -394,4 +490,227 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
-end
+  def test_emit_prometheus_with_custom_dimensions
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_dimensions   'foo=bar, dolor=sit,amet,test'
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'X-Sumo-Dimensions'=>'foo=bar, dolor=sit',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
+  def test_emit_prometheus_with_empty_custom_metadata
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_metadata     "    "
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
+  def test_batching_same_headers
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      json
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1'})
+      driver.feed("output.test", time, {'message' => 'test2'})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\n{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+  end
+
+  def test_batching_different_headers
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      json
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1', '_sumo_metadata' => {"category": "cat1"}})
+      driver.feed("output.test", time, {'message' => 'test2', '_sumo_metadata' => {"category": "cat2"}})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'cat1', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'cat2', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+  end
+
+  def test_batching_different_fields
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1'})
+      driver.feed("output.test", time, {'message' => 'test2', '_sumo_metadata' => {"fields": "foo=bar"}})
+      driver.feed("output.test", time, {'message' => 'test3', '_sumo_metadata' => {"fields": "foo=bar,sumo=logic"}})
+      driver.feed("output.test", time, {'message' => 'test4', '_sumo_metadata' => {"fields": "foo=bar,master_url=https://100.64.0.1:443"}})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar'},
+                      body: /\A{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar,sumo=logic'},
+                      body: /\A{"timestamp":\d+.,"message":"test3"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar,master_url=https://100.64.0.1:443'},
+                      body: /\A{"timestamp":\d+.,"message":"test4"}\z/,
+                      times:1
+  end
+
+  def test_emit_json_merge_timestamp_compress_deflate
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+      compress_encoding deflate
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":123}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'deflate'},
+                     body: "\x78\x9c\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\xae\x05\x00\x38\xb0\x05\xe1".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
+  def test_emit_json_merge_timestamp_compress_gzip
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":1234}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'gzip'},
+                     body: "\x1f\x8b\x08\x00\x01\x00\x00\x00\x00\x03\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\x36\xa9\x05\x00\xfe\x53\xbe\x14\x12\x00\x00\x00".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
+  def test_emit_text_from_array
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => ['test', 'test2']})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '["test","test2"]',
+                     times:1
+  end
+
+  def test_emit_text_from_dict
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => {'test': 'test2', 'test3': 'test4'}})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '{"test":"test2","test3":"test4"}',
+                     times:1
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.7.2
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-26 00:00:00.000000000 Z
+date: 2020-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint