fluent-plugin-sumologic_output 1.6.1 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec49fe6284202d7f57bdd3b59b06f5248d5345e84b7fd955f69b47c069db28e6
-  data.tar.gz: 80389b25c291df5258a4313f1025e5e84507207f5f11911b768ecc273391738b
+  metadata.gz: 52b0c77e900d61ba360421933440383208033b29b0f08b012250519189872f24
+  data.tar.gz: ff86f952744e650e30ba94d1185f4d270a3d8d6e7bfee5f30a0810e043a9483f
 SHA512:
-  metadata.gz: 3fb838f98317572424ad7969450b4f7811a94a80de123e39ce8da9ff0dc9fb1448e1262686e930260142816732ae0ce0420b9b3b52f9d9ade5807e86951dd9e0
-  data.tar.gz: 8a19573a03b6dd156fa25f74aeff3adb9059877f5ffcd1701c25f3a359d83e04668d9647c70dc1eb47f3e5cf3da6c2b6e3cb57c2ebec8cf9205d64b3e911bbe5
+  metadata.gz: 74c43d262f35e2767d170a05dbec58911590c0851a724f6c4adb0047a26601383e4d8fa5e2671c845bbd89a65781fb0d35e4bc2f8d7ed6bd0f0e376d6c7ec958
+  data.tar.gz: 73b8ea2a6dbf4593eca7d4a702eeea14f27728bd7aa28ddc15e968301a5a2eb21f76ae18f1be2dd759f42cb357c2ceda92aec49e136755d76c954ebded889cdd

data/.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Learn about CODEOWNERS file format:
+#  https://help.github.com/en/articles/about-code-owners
+
+* @SumoLogic/open-source-collection-team
+

data/.gitignore

@@ -2,4 +2,5 @@
 *.rbc
 Gemfile.lock
 .idea/
-TAGS
+TAGS
+.vagrant

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.7.1"></a>
+# [1.7.1] (2020-04-28)
+- Fix configuration for older fluentd versions [#63](https://github.com/SumoLogic/fluentd-output-sumologic/pull/63)
+
+<a name="1.7.0"></a>
+# [1.7.0] (2020-04-23)
+- Add option for specifing custom fields for logs: [#56](https://github.com/SumoLogic/fluentd-output-sumologic/pull/56)
+- Add option for specifing custom dimensions for metrics: [#57](https://github.com/SumoLogic/fluentd-output-sumologic/pull/57)
+- Add support for compression: [#58](https://github.com/SumoLogic/fluentd-output-sumologic/pull/58)
+
 <a name="1.5.0"></a>
 # [1.5.0] (2019-06-26)
 - Add support for new log format fields: [#49](https://github.com/SumoLogic/fluentd-output-sumologic/pull/49)

data/README.md

@@ -4,9 +4,6 @@
 
 This plugin has been designed to output logs or metrics to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source)
 
-## Support
-The code in this repository has been developed in collaboration with the Sumo Logic community and is not supported via standard Sumo Logic Support channels. For any issues or questions please submit an issue within the GitHub repository. The maintainers of this project will work directly with the community to answer any questions, address bugs, or review any requests for new features. 
-
 ## License
 Released under Apache 2.0 License. 
 
@@ -31,11 +28,16 @@ Configuration options for fluent.conf are:
   * json_merge - Same as json but merge content of `log_key` into the top level and strip `log_key`
 * `log_key` - Used to specify the key when merging json or sending logs in text format (default `message`)
 * `open_timeout` - Set timeout seconds to wait until connection is opened.
+* `send_timeout` - Timeout for sending to SumoLogic in seconds. Don't modify unless you see `HTTPClient::SendTimeoutError` in your Fluentd logs. (default `120`)
 * `add_timestamp` - Add `timestamp` (or `timestamp_key`) field to logs before sending to sumologic (default `true`)
 * `timestamp_key` - Field name when `add_timestamp` is on (default `timestamp`)
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
 * `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
 * `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
+* `compress` - Option to enable compression (default `false`)
+* `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
+* `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
+* `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 

data/Vagrantfile

@@ -0,0 +1,27 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+unless Vagrant.has_plugin?("vagrant-disksize")
+  puts "vagrant-disksize plugin unavailable\n" +
+       "please install it via 'vagrant plugin install vagrant-disksize'"
+  exit 1
+end
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'ubuntu/focal64'
+  config.disksize.size = '50GB'
+  config.vm.box_check_update = false
+  config.vm.host_name = 'fluentd-output-sumologic'
+  config.vm.network :private_network, ip: "192.168.78.45"
+
+  config.vm.provider 'virtualbox' do |vb|
+    vb.gui = false
+    vb.cpus = 8
+    vb.memory = 16384
+    vb.name = 'fluentd-output-sumologic'
+  end
+
+  config.vm.provision 'shell', path: 'vagrant/provision.sh'
+
+  config.vm.synced_folder ".", "/sumologic"
+end

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.6.1"
+  gem.version       = "1.7.3"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -1,31 +1,49 @@
 require 'fluent/plugin/output'
 require 'net/https'
+require 'json'
 require 'yajl'
 require 'httpclient'
+require 'zlib'
+require 'stringio'
 
 class SumologicConnection
 
   attr_reader :http
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  COMPRESS_DEFLATE = 'deflate'
+  COMPRESS_GZIP = 'gzip'
+
+  def initialize(endpoint, verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
     @endpoint = endpoint
-    create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    @sumo_client = sumo_client
+    create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
+    @compress = compress_enabled
+    @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+
+    unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
+      raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
+    end
   end
 
-  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields)
-    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields))
+  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields, dimensions)
+    response = http.post(@endpoint, compress(raw_data), request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields, dimensions))
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
   end
 
-  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields)
+  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
     headers = {
         'X-Sumo-Name'     => source_name,
         'X-Sumo-Category' => source_category,
         'X-Sumo-Host'     => source_host,
-        'X-Sumo-Client'   => 'fluentd-output'
+        'X-Sumo-Client'   => @sumo_client,
     }
+
+    if @compress
+      headers['Content-Encoding'] = @compress_encoding
+    end
+
     if data_type == 'metrics'
       case metric_data_format
       when 'graphite'
@@ -37,6 +55,10 @@ class SumologicConnection
       else
         raise RuntimeError, "Invalid #{metric_data_format}, must be graphite or carbon2 or prometheus"
       end
+
+      unless dimensions.nil?
+        headers['X-Sumo-Dimensions'] = dimensions
+      end
     end
     unless collected_fields.nil?
       headers['X-Sumo-Fields'] = collected_fields
@@ -48,14 +70,38 @@ class SumologicConnection
     verify_ssl==true ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
   end
 
-  def create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  def create_http_client(verify_ssl, connect_timeout, send_timeout, proxy_uri, disable_cookies)
     @http                        = HTTPClient.new(proxy_uri)
     @http.ssl_config.verify_mode = ssl_options(verify_ssl)
     @http.connect_timeout        = connect_timeout
+    @http.send_timeout           = send_timeout
     if disable_cookies
       @http.cookie_manager       = nil
     end
   end
+
+  def compress(content)
+    if @compress
+      if @compress_encoding == COMPRESS_GZIP
+        result = gzip(content)
+        result.bytes.to_a.pack("c*")
+      else
+        Zlib::Deflate.deflate(content)
+      end
+    else
+      content
+    end
+  end # def compress
+  
+  def gzip(content)
+    stream = StringIO.new("w")
+    stream.set_encoding("ASCII")
+    gz = Zlib::GzipWriter.new(stream)
+    gz.mtime=1  # Ensure that for same content there is same output
+    gz.write(content)
+    gz.close
+    stream.string.bytes.to_a.pack("c*")
+  end # def gzip
 end
 
 class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
@@ -82,10 +128,23 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :verify_ssl, :bool, :default => true
   config_param :delimiter, :string, :default => "."
   config_param :open_timeout, :integer, :default => 60
+  config_param :send_timeout, :integer, :default => 120
   config_param :add_timestamp, :bool, :default => true
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+  # https://help.sumologic.com/Manage/Fields
+  desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
+  config_param :custom_fields, :string, :default => nil
+  desc 'Name of sumo client which is send as X-Sumo-Client header'
+  config_param :sumo_client, :string, :default => 'fluentd-output'
+  desc 'Compress payload'
+  config_param :compress, :bool, :default => false
+  desc 'Encoding method of compresssion (either gzip or deflate)'
+  config_param :compress_encoding, :string, :default => SumologicConnection::COMPRESS_GZIP
+  # https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers
+  desc 'Dimensions string (eg "cluster=payment, service=credit_card") which is going to be added to every metric record.'
+  config_param :custom_dimensions, :string, :default => nil
 
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -129,7 +188,38 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       end
     end
 
-    @sumo_conn = SumologicConnection.new(conf['endpoint'], conf['verify_ssl'], conf['open_timeout'].to_i, conf['proxy_uri'], conf['disable_cookies'])
+    conf['custom_fields'] = validate_key_value_pairs(conf['custom_fields'])
+    if conf['custom_fields'].nil?
+      conf.delete 'custom_fields'
+    end
+    unless conf['custom_fields']
+      @log.debug "Custom fields: #{conf['custom_fields']}"
+    end
+
+    conf['custom_dimensions'] = validate_key_value_pairs(conf['custom_dimensions'])
+    if conf['custom_dimensions'].nil?
+      conf.delete 'custom_dimensions'
+    end
+    unless conf['custom_dimensions']
+      @log.debug "Custom dimensions: #{conf['custom_dimensions']}"
+    end
+
+    # For some reason default is set incorrectly in unit-tests
+    if conf['sumo_client'].nil? || conf['sumo_client'].strip.length == 0
+      conf['sumo_client'] = 'fluentd-output'
+    end
+
+    @sumo_conn = SumologicConnection.new(
+      conf['endpoint'],
+      conf['verify_ssl'],
+      conf['open_timeout'].to_i,
+      conf['send_timeout'].to_i,
+      conf['proxy_uri'],
+      conf['disable_cookies'],
+      conf['sumo_client'],
+      conf['compress'],
+      conf['compress_encoding']
+      )
     super
   end
 
@@ -163,8 +253,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   def dump_log(log)
     log.delete('_sumo_metadata')
     begin
-      parser = Yajl::Parser.new
-      hash = parser.parse(log[@log_key])
+      hash = JSON.parse(log[@log_key])
       log[@log_key] = hash
       Yajl.dump(log)
     rescue
@@ -207,6 +296,19 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     time.to_s.length == 13 ? time : time * 1000
   end
 
+  # Convert log to string and strip it
+  def log_to_str(log)
+    if log.is_a?(Array) or log.is_a?(Hash)
+      log = Yajl.dump(log)
+    end
+
+    unless log.nil?
+      log.strip!
+    end
+
+    return log
+  end
+
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
     messages_list = {}
@@ -227,10 +329,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       when 'logs'
         case log_format
         when 'text'
-          log = record[@log_key]
-          unless log.nil?
-            log.strip!
-          end
+          log = log_to_str(record[@log_key])
         when 'json_merge'
           if @add_timestamp
             record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
@@ -248,10 +347,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           log = dump_log(record)
         end
       when 'metrics'
-        log = record[@log_key]
-        unless log.nil?
-          log.strip!
-        end
+        log = log_to_str(record[@log_key])
       end
 
       unless log.nil?
@@ -268,6 +364,14 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     messages_list.each do |key, messages|
       source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
         key[:source_host], key[:fields]
+
+      # Merge custom and record fields
+      if fields.nil? || fields.strip.length == 0
+        fields = @custom_fields
+      else
+        fields = [fields,@custom_fields].compact.join(",")
+      end
+
       @sumo_conn.publish(
           messages.join("\n"),
           source_host         =source_host,
@@ -275,9 +379,26 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           source_name         =source_name,
           data_type           =@data_type,
           metric_data_format  =@metric_data_format,
-          collected_fields    =fields
+          collected_fields    =fields,
+          dimensions          =@custom_dimensions
       )
     end
 
   end
+
+  def validate_key_value_pairs(fields)
+    if fields.nil?
+      return fields
+    end
+
+    fields = fields.split(",").select { |field|
+      field.split('=').length == 2
+    }
+
+    if fields.length == 0
+      return nil
+    end
+
+    fields.join(',')
+  end
 end

data/test/plugin/test_out_sumologic.rb

@@ -72,6 +72,8 @@ class SumologicOutput < Test::Unit::TestCase
     assert_equal instance.timestamp_key, 'timestamp'
     assert_equal instance.proxy_uri, nil
     assert_equal instance.disable_cookies, false
+    assert_equal instance.sumo_client, 'fluentd-output'
+    assert_equal instance.compress_encoding, 'gzip'
   end
 
   def test_emit_text
@@ -95,6 +97,28 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_text_custom_sumo_client
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+      sumo_client     'fluentd-custom-sender'
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => 'test'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-custom-sender', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: "test",
+                     times:1
+  end
+
   def test_emit_json
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -267,6 +291,78 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_with_sumo_metadata_with_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,dolor=amet"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic,lorem=ipsum,dolor=amet'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_fields_and_empty_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   ""
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_empty_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,invalid"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": ""
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'lorem=ipsum'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
   def test_emit_with_sumo_metadata
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -394,6 +490,61 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_prometheus_with_custom_dimensions
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_dimensions   'foo=bar, dolor=sit,amet,test'
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'X-Sumo-Dimensions'=>'foo=bar, dolor=sit',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
+  def test_emit_prometheus_with_empty_custom_metadata
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_metadata     "    "
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
   def test_batching_same_headers
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -475,4 +626,175 @@ class SumologicOutput < Test::Unit::TestCase
                       times:1
   end
 
-end
+  def test_emit_json_merge_timestamp_compress_deflate
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+      compress_encoding deflate
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":123}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'deflate'},
+                     body: "\x78\x9c\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\xae\x05\x00\x38\xb0\x05\xe1".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
+  def test_emit_json_merge_timestamp_compress_gzip
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":1234}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'gzip'},
+                     body: "\x1f\x8b\x08\x00\x01\x00\x00\x00\x00\x03\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\x36\xa9\x05\x00\xfe\x53\xbe\x14\x12\x00\x00\x00".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
+  def test_emit_text_from_array
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => ['test', 'test2']})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '["test","test2"]',
+                     times:1
+  end
+
+  def test_emit_text_from_dict
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => {'test': 'test2', 'test3': 'test4'}})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: '{"test":"test2","test3":"test4"}',
+                     times:1
+  end
+
+  def test_emit_fields_string_based
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":{"foo":"bar","message":"test"}}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_1
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message": "test"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\": \\"test\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_2
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"foo": "bar", "message"'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"{\\"foo\\": \\"bar\\", \\"message\\""}\z/,
+                     times:1
+  end
+  
+  def test_emit_fields_invalid_json_string_based_3
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+  
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '"foo\": \"bar\", \"mess'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"\\"foo\\\\\\": \\\\\\"bar\\\\\\", \\\\\\"mess"}\z/,
+                     times:1
+  end  
+
+end

data/vagrant/README.md

@@ -0,0 +1,43 @@
+# Vagrant
+
+## Prerequisites
+
+Please install the following:
+
+- [VirtualBox](https://www.virtualbox.org/)
+- [Vagrant](https://www.vagrantup.com/)
+- [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin
+
+### MacOS
+
+```bash
+brew cask install virtualbox
+brew cask install vagrant
+vagrant plugin install vagrant-disksize
+```
+
+## Setting up
+
+You can set up the Vagrant environment with just one command:
+
+```bash
+vagrant up
+```
+
+After successfull installation you can ssh to the virtual machine with:
+
+```bash
+vagrant ssh
+```
+
+NOTICE: The directory with fluentd-output-sumologic repository on the host is synced with `/sumologic/` directory on the virtual machine.
+
+## Runing tests
+
+You can run tests using following commands:
+
+```bash
+cd /sumologic
+bundle install
+bundle exec rake
+```

data/vagrant/provision.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get --yes upgrade
+apt-get --yes install apt-transport-https
+
+echo "export EDITOR=vim" >> /home/vagrant/.bashrc
+
+# Install docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+apt-get install -y docker-ce docker-ce-cli containerd.io
+usermod -aG docker vagrant
+
+# Install make
+apt-get install -y make
+
+# install requirements for ruby
+snap install ruby --channel=2.6/stable --classic
+su vagrant -c 'gem install bundler:2.1.4'
+apt install -y gcc g++ libsnappy-dev libicu-dev zlib1g-dev cmake pkg-config libssl-dev

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.3
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2021-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -103,6 +103,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -110,11 +111,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- Vagrantfile
 - ci/build.sh
 - fluent-plugin-sumologic_output.gemspec
 - lib/fluent/plugin/out_sumologic.rb
 - test/helper.rb
 - test/plugin/test_out_sumologic.rb
+- vagrant/README.md
+- vagrant/provision.sh
 homepage: https://github.com/SumoLogic/fluentd-output-sumologic
 licenses:
 - Apache-2.0
@@ -134,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint