fluent-plugin-sumologic_output 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6db47bbad47c3eb7ea578b1e894ddc972336e785c94bb8af47595481b7e934e2
-  data.tar.gz: 3457257200adc1b655ed1820a5041e29c22531e621aa24a6dd620105ac4559f1
+  metadata.gz: 133846d8832ef986550e20aa4d830f1b38c22528398ed4ef589af16cf553be2e
+  data.tar.gz: 6dde4d581727f2c6703b0e31d1907b935f53dd744b2b133ba36ca8ba437338c2
 SHA512:
-  metadata.gz: cad8b409225fdd0d96402c17a950c0fb67ad61d0f28ac98a252bac23671cee9a3152738c37f916295c116c0396ede9ea874339c8d4278daf81f698060b56f63d
-  data.tar.gz: 619b7cc028a11d1485b158a08e5f34f00440e5410b14b4283d0dda778f80d22944be9f753275a3028c360e15a69baf437b761a09c7e6b29e2f2c784bf3c0d965
+  metadata.gz: d9f7bb3b6a59030a406eaac3c69568f03c98c4dd00578f1a089687e7d8e7d2f2a5e3f1ce7d76eaecda0ea26a8dd4519f0ee8ea2533b85bcdc9f75fdca0b703f0
+  data.tar.gz: 0d7074aeb0dfcca743cce825690e4b5fd03f3c13e01df0bfb94b85762639747d51b1c7434aa15b84ea83db790507fd058b16098ec01f42eb6ce236dca57407c2

data/CHANGELOG.md

@@ -2,7 +2,11 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
-<a name="1.4.0"></a>
+<a name="1.5.0"></a>
+# [1.5.0] (2019-06-26)
+- Add support for new log format fields: [#49](https://github.com/SumoLogic/fluentd-output-sumologic/pull/49)
+
+<a name="1.4.1"></a>
 # [1.4.1] (2019-03-13)
 
 - Add option for sending metrics in Prometheus format [#39](https://github.com/SumoLogic/fluentd-output-sumologic/pull/39) 

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.5.0"
+  gem.version       = "1.6.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -195,7 +195,10 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     source_host = sumo_metadata['host'] || @source_host
     source_host = extract_placeholders(source_host, chunk) unless source_host.nil?
 
-    "#{source_name}:#{source_category}:#{source_host}"
+    fields = sumo_metadata['fields'] || ""
+    fields = extract_placeholders(fields, chunk) unless fields.nil?
+
+    "#{source_name}:#{source_category}:#{source_host}:#{fields}"
   end
 
   # Convert timestamp to 13 digit epoch if necessary
@@ -203,28 +206,9 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     time.to_s.length == 13 ? time : time * 1000
   end
 
-  def sumo_fields(sumo_metadata)
-    fields = sumo_metadata['fields'] || ""
-    Hash[
-        fields.split(',').map do |pair|
-          k, v = pair.split('=', 2)
-          [k, v]
-        end
-    ]
-  end
-
-  def dump_collected_fields(log_fields)
-    if log_fields.nil?
-      log_fields
-    else
-      log_fields.map{|k,v| "#{k}=#{v}"}.join(',')
-    end
-  end
-
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
     messages_list = {}
-    log_fields = nil
 
     # Sort messages
     chunk.msgpack_each do |time, record|
@@ -252,7 +236,6 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           end
           log = dump_log(merge_json(record))
         when 'fields'
-          log_fields = sumo_fields(sumo_metadata)
           if @add_timestamp
             record = {  @timestamp_key => sumo_timestamp(time) }.merge(record)
           end
@@ -282,7 +265,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     # Push logs to sumo
     messages_list.each do |key, messages|
-      source_name, source_category, source_host = key.split(':')
+      source_name, source_category, source_host, fields = key.split(':')
       @sumo_conn.publish(
           messages.join("\n"),
           source_host         =source_host,
@@ -290,7 +273,7 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           source_name         =source_name,
           data_type           =@data_type,
           metric_data_format  =@metric_data_format,
-          collected_fields    =dump_collected_fields(log_fields)
+          collected_fields    =fields
       )
     end
 

data/test/plugin/test_out_sumologic.rb

@@ -394,4 +394,85 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_batching_same_headers
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      json
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1'})
+      driver.feed("output.test", time, {'message' => 'test2'})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\n{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+  end
+
+  def test_batching_different_headers
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      json
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1', '_sumo_metadata' => {"category": "cat1"}})
+      driver.feed("output.test", time, {'message' => 'test2', '_sumo_metadata' => {"category": "cat2"}})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'cat1', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'cat2', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+  end
+
+  def test_batching_different_fields
+    config = %{
+      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+      log_format      fields
+      source_category test
+      source_host     test
+      source_name     test
+    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test1'})
+      driver.feed("output.test", time, {'message' => 'test2', '_sumo_metadata' => {"fields": "foo=bar"}})
+      driver.feed("output.test", time, {'message' => 'test3', '_sumo_metadata' => {"fields": "foo=bar,sumo=logic"}})
+      driver.feed("output.test", time, {'message' => 'test4', '_sumo_metadata' => {"fields": "foo=bar,abc=123"}})
+    end
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                      body: /\A{"timestamp":\d+.,"message":"test1"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar'},
+                      body: /\A{"timestamp":\d+.,"message":"test2"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar,sumo=logic'},
+                      body: /\A{"timestamp":\d+.,"message":"test3"}\z/,
+                      times:1
+    assert_requested  :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                      headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test', 'X-Sumo-Fields' => 'foo=bar,abc=123'},
+                      body: /\A{"timestamp":\d+.,"message":"test4"}\z/,
+                      times:1
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-26 00:00:00.000000000 Z
+date: 2019-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,7 +134,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint