fluent-plugin-sumologic_output 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec49fe6284202d7f57bdd3b59b06f5248d5345e84b7fd955f69b47c069db28e6
-  data.tar.gz: 80389b25c291df5258a4313f1025e5e84507207f5f11911b768ecc273391738b
+  metadata.gz: 7d4b2b4d4ebc9a15dc335f7fba1ca912981caea13834a7007ab46788fe15c5c0
+  data.tar.gz: dec9d40e136d1e1662c2b46c890135d109b202edb6f9836bfad8f83463b3372f
 SHA512:
-  metadata.gz: 3fb838f98317572424ad7969450b4f7811a94a80de123e39ce8da9ff0dc9fb1448e1262686e930260142816732ae0ce0420b9b3b52f9d9ade5807e86951dd9e0
-  data.tar.gz: 8a19573a03b6dd156fa25f74aeff3adb9059877f5ffcd1701c25f3a359d83e04668d9647c70dc1eb47f3e5cf3da6c2b6e3cb57c2ebec8cf9205d64b3e911bbe5
+  metadata.gz: 25fd5bbe86e7a150e68cc1376029a622276cc8e9f808add7da6fdd18a2fc00419a1899798a50518a717d2ac8615be8a51cde5321036883fb2995a15fa27967ed
+  data.tar.gz: 84c7344ddab12e2ee78497a4e07bea05f1134a8dad44bc4042a0a9032f12d6605d82f4c5be366143f3b175fe113af17ee1f034946406dd1eeb8fedb5aefe4ec2

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.7.0"></a>
+# [1.7.0] (2020-04-23)
+- Add option for specifing custom fields for logs: [#56](https://github.com/SumoLogic/fluentd-output-sumologic/pull/56)
+- Add option for specifing custom dimensions for metrics: [#57](https://github.com/SumoLogic/fluentd-output-sumologic/pull/57)
+- Add support for compression: [#58](https://github.com/SumoLogic/fluentd-output-sumologic/pull/58)
+
 <a name="1.5.0"></a>
 # [1.5.0] (2019-06-26)
 - Add support for new log format fields: [#49](https://github.com/SumoLogic/fluentd-output-sumologic/pull/49)

data/README.md

@@ -4,9 +4,6 @@
 
 This plugin has been designed to output logs or metrics to [SumoLogic](http://www.sumologic.com) via a [HTTP collector endpoint](http://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source)
 
-## Support
-The code in this repository has been developed in collaboration with the Sumo Logic community and is not supported via standard Sumo Logic Support channels. For any issues or questions please submit an issue within the GitHub repository. The maintainers of this project will work directly with the community to answer any questions, address bugs, or review any requests for new features. 
-
 ## License
 Released under Apache 2.0 License. 
 
@@ -36,6 +33,10 @@ Configuration options for fluent.conf are:
 * `proxy_uri` - Add the `uri` of the `proxy` environment if present.
 * `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
 * `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
+* `compress` - Option to enable compression (default `false`)
+* `compress_encoding` - Compression encoding format, either `gzip` or `deflate` (default `gzip`)
+* `custom_fields` - Comma-separated key=value list of fields to apply to every log. [more information](https://help.sumologic.com/Manage/Fields#http-source-fields)
+* `custom_dimensions` - Comma-separated key=value list of dimensions to apply to every metric. [more information](https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers)
 
 __NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
 

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.6.1"
+  gem.version       = "1.7.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -2,30 +2,47 @@ require 'fluent/plugin/output'
 require 'net/https'
 require 'yajl'
 require 'httpclient'
+require 'zlib'
+require 'stringio'
 
 class SumologicConnection
 
   attr_reader :http
 
-  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+  COMPRESS_DEFLATE = 'deflate'
+  COMPRESS_GZIP = 'gzip'
+
+  def initialize(endpoint, verify_ssl, connect_timeout, proxy_uri, disable_cookies, sumo_client, compress_enabled, compress_encoding)
     @endpoint = endpoint
+    @sumo_client = sumo_client
     create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
+    @compress = compress_enabled
+    @compress_encoding = (compress_encoding ||= COMPRESS_GZIP).downcase
+
+    unless [COMPRESS_DEFLATE, COMPRESS_GZIP].include? @compress_encoding
+      raise "Invalid compression encoding #{@compress_encoding} must be gzip or deflate"
+    end
   end
 
-  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields)
-    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields))
+  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields, dimensions)
+    response = http.post(@endpoint, compress(raw_data), request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields, dimensions))
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
   end
 
-  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields)
+  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields, dimensions)
     headers = {
         'X-Sumo-Name'     => source_name,
         'X-Sumo-Category' => source_category,
         'X-Sumo-Host'     => source_host,
-        'X-Sumo-Client'   => 'fluentd-output'
+        'X-Sumo-Client'   => @sumo_client,
     }
+
+    if @compress
+      headers['Content-Encoding'] = @compress_encoding
+    end
+
     if data_type == 'metrics'
       case metric_data_format
       when 'graphite'
@@ -37,6 +54,10 @@ class SumologicConnection
       else
         raise RuntimeError, "Invalid #{metric_data_format}, must be graphite or carbon2 or prometheus"
       end
+
+      unless dimensions.nil?
+        headers['X-Sumo-Dimensions'] = dimensions
+      end
     end
     unless collected_fields.nil?
       headers['X-Sumo-Fields'] = collected_fields
@@ -56,6 +77,29 @@ class SumologicConnection
       @http.cookie_manager       = nil
     end
   end
+
+  def compress(content)
+    if @compress
+      if @compress_encoding == COMPRESS_GZIP
+        result = gzip(content)
+        result.bytes.to_a.pack("c*")
+      else
+        Zlib::Deflate.deflate(content)
+      end
+    else
+      content
+    end
+  end # def compress
+  
+  def gzip(content)
+    stream = StringIO.new("w")
+    stream.set_encoding("ASCII")
+    gz = Zlib::GzipWriter.new(stream)
+    gz.mtime=1  # Ensure that for same content there is same output
+    gz.write(content)
+    gz.close
+    stream.string.bytes.to_a.pack("c*")
+  end # def gzip
 end
 
 class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
@@ -86,6 +130,18 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   config_param :timestamp_key, :string, :default => 'timestamp'
   config_param :proxy_uri, :string, :default => nil
   config_param :disable_cookies, :bool, :default => false
+  # https://help.sumologic.com/Manage/Fields
+  desc 'Fields string (eg "cluster=payment, service=credit_card") which is going to be added to every log record.'
+  config_param :custom_fields, :string, :default => nil
+  desc 'Name of sumo client which is send as X-Sumo-Client header'
+  config_param :sumo_client, :string, :default => 'fluentd-output'
+  desc 'Compress payload'
+  config_param :compress, :bool, :default => false
+  desc 'Encoding method of compresssion (either gzip or deflate)'
+  config_param :compress_encoding, :string, :default => SumologicConnection::COMPRESS_GZIP
+  # https://help.sumologic.com/03Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/Upload-Metrics-to-an-HTTP-Source#supported-http-headers
+  desc 'Dimensions string (eg "cluster=payment, service=credit_card") which is going to be added to every metric record.'
+  config_param :custom_dimensions, :string, :default => nil
 
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -129,7 +185,31 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
       end
     end
 
-    @sumo_conn = SumologicConnection.new(conf['endpoint'], conf['verify_ssl'], conf['open_timeout'].to_i, conf['proxy_uri'], conf['disable_cookies'])
+    conf['custom_fields'] = validate_key_value_pairs(conf['custom_fields'])
+    unless conf['custom_fields']
+      @log.info "Custom fields: #{conf['custom_fields']}"
+    end
+
+    conf['custom_dimensions'] = validate_key_value_pairs(conf['custom_dimensions'])
+    unless conf['custom_dimensions']
+      @log.info "Custom dimensions: #{conf['custom_dimensions']}"
+    end
+
+    # For some reason default is set incorrectly in unit-tests
+    if conf['sumo_client'].nil? || conf['sumo_client'].strip.length == 0
+      conf['sumo_client'] = 'fluentd-output'
+    end
+
+    @sumo_conn = SumologicConnection.new(
+      conf['endpoint'],
+      conf['verify_ssl'],
+      conf['open_timeout'].to_i,
+      conf['proxy_uri'],
+      conf['disable_cookies'],
+      conf['sumo_client'],
+      conf['compress'],
+      conf['compress_encoding']
+      )
     super
   end
 
@@ -268,6 +348,14 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     messages_list.each do |key, messages|
       source_name, source_category, source_host, fields = key[:source_name], key[:source_category],
         key[:source_host], key[:fields]
+
+      # Merge custom and record fields
+      if fields.nil? || fields.strip.length == 0
+        fields = @custom_fields
+      else
+        fields = [fields,@custom_fields].compact.join(",")
+      end
+
       @sumo_conn.publish(
           messages.join("\n"),
           source_host         =source_host,
@@ -275,9 +363,26 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           source_name         =source_name,
           data_type           =@data_type,
           metric_data_format  =@metric_data_format,
-          collected_fields    =fields
+          collected_fields    =fields,
+          dimensions          =@custom_dimensions
       )
     end
 
   end
+
+  def validate_key_value_pairs(fields)
+    if fields.nil?
+      return fields
+    end
+
+    fields = fields.split(",").select { |field|
+      field.split('=').length == 2
+    }
+
+    if fields.length == 0
+      return nil
+    end
+
+    fields.join(',')
+  end
 end

data/test/plugin/test_out_sumologic.rb

@@ -72,6 +72,8 @@ class SumologicOutput < Test::Unit::TestCase
     assert_equal instance.timestamp_key, 'timestamp'
     assert_equal instance.proxy_uri, nil
     assert_equal instance.disable_cookies, false
+    assert_equal instance.sumo_client, 'fluentd-output'
+    assert_equal instance.compress_encoding, 'gzip'
   end
 
   def test_emit_text
@@ -95,6 +97,28 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_text_custom_sumo_client
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      text
+      source_category test
+      source_host     test
+      source_name     test
+      sumo_client     'fluentd-custom-sender'
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => 'test'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-custom-sender', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: "test",
+                     times:1
+  end
+
   def test_emit_json
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -267,6 +291,78 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_with_sumo_metadata_with_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,dolor=amet"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic,lorem=ipsum,dolor=amet'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_fields_and_empty_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   ""
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_empty_fields_and_custom_fields_fields_format
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      custom_fields   "lorem=ipsum,invalid"
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": ""
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'lorem=ipsum'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
   def test_emit_with_sumo_metadata
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -394,6 +490,61 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_prometheus_with_custom_dimensions
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_dimensions   'foo=bar, dolor=sit,amet,test'
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'X-Sumo-Dimensions'=>'foo=bar, dolor=sit',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
+  def test_emit_prometheus_with_empty_custom_metadata
+    config = %{
+      endpoint            https://collectors.sumologic.com/v1/receivers/http/1234
+      data_type           metrics
+      metric_data_format  prometheus
+      source_category     test
+      source_host         test
+      source_name         test
+      custom_metadata     "    "
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' =>'cpu{cluster="prod", node="lb-1"} 87.2 1501753030'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {
+                       'X-Sumo-Category'=>'test',
+                       'X-Sumo-Client'=>'fluentd-output',
+                       'X-Sumo-Host'=>'test',
+                       'X-Sumo-Name'=>'test',
+                       'Content-Type'=>'application/vnd.sumologic.prometheus'},
+                     body: 'cpu{cluster="prod", node="lb-1"} 87.2 1501753030',
+                     times:1
+  end
+
   def test_batching_same_headers
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
@@ -475,4 +626,49 @@ class SumologicOutput < Test::Unit::TestCase
                       times:1
   end
 
+  def test_emit_json_merge_timestamp_compress_deflate
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+      compress_encoding deflate
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":123}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'deflate'},
+                     body: "\x78\x9c\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\xae\x05\x00\x38\xb0\x05\xe1".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
+  def test_emit_json_merge_timestamp_compress_gzip
+    config = %{
+      endpoint          https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format        json_merge
+      source_category   test
+      source_host       test
+      source_name       test
+      compress          true
+
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => '{"timestamp":1234}'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'Content-Encoding'=>'gzip'},
+                     body: "\x1f\x8b\x08\x00\x01\x00\x00\x00\x00\x03\xab\x56\x2a\xc9\xcc\x4d\x2d\x2e\x49\xcc\x2d\x50\xb2\x32\x34\x32\x36\xa9\x05\x00\xfe\x53\xbe\x14\x12\x00\x00\x00".force_encoding("ASCII-8BIT"),
+                     times:1
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2020-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler