fluent-plugin-sumologic_output 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4346ff078527905ff2b54424eef12afbbb88b37174fdca0e7b36cb6addadf2a7
-  data.tar.gz: 83bfd102b85451d7f4ec5d6635482efbdd945fe8ef2e52d14bfd56ca1b48cb18
+  metadata.gz: 6db47bbad47c3eb7ea578b1e894ddc972336e785c94bb8af47595481b7e934e2
+  data.tar.gz: 3457257200adc1b655ed1820a5041e29c22531e621aa24a6dd620105ac4559f1
 SHA512:
-  metadata.gz: 6ac64815e9d60e5ecd9af2fb867e28822185b12d368ed0a3566bdfc0f0a958dc77d2ae5bef11cefba13306f4691a5b72e892c28b9a7e8282fb4193c645799d43
-  data.tar.gz: 27623a54343eab0c96f7c1e8577176cc6deccffcd65d34af7d9411b0334051d365bfe6ea9e7bae34462163282cc84af86bded65300130ff908a0b47d86c9f63a
+  metadata.gz: cad8b409225fdd0d96402c17a950c0fb67ad61d0f28ac98a252bac23671cee9a3152738c37f916295c116c0396ede9ea874339c8d4278daf81f698060b56f63d
+  data.tar.gz: 619b7cc028a11d1485b158a08e5f34f00440e5410b14b4283d0dda778f80d22944be9f753275a3028c360e15a69baf437b761a09c7e6b29e2f2c784bf3c0d965

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
 
+<a name="1.4.0"></a>
+# [1.4.1] (2019-03-13)
+
+- Add option for sending metrics in Prometheus format [#39](https://github.com/SumoLogic/fluentd-output-sumologic/pull/39) 
+- Use the build-in extract_placeholders method for header expanding [#40](https://github.com/SumoLogic/fluentd-output-sumologic/pull/40)
+
+__NOTE:__ there is a breaking change in the placeholders: `tag_parts[n]` is replaced by `tag[n]` [#47](https://github.com/SumoLogic/fluentd-output-sumologic/issues/47)
+
 <a name="1.4.0"></a>
 # [1.4.0] (2019-01-16)
 

data/README.md

@@ -21,10 +21,10 @@ Configuration options for fluent.conf are:
 * `data_type` - The type of data that will be sent to Sumo Logic, either `logs` or `metrics` (Default is `logs `)
 * `endpoint` - SumoLogic HTTP Collector URL
 * `verify_ssl` - Verify ssl certificate. (default is `true`)
-* `source_category` - Set _sourceCategory metadata field within SumoLogic (default is `nil`)
-* `source_name` - Set _sourceName metadata field within SumoLogic - overrides source_name_key (default is `nil`)
+* `source_category`<sup>*</sup> - Set _sourceCategory metadata field within SumoLogic (default is `nil`)
+* `source_name`<sup>*</sup> - Set _sourceName metadata field within SumoLogic - overrides source_name_key (default is `nil`)
 * `source_name_key` - Set as source::path_key's value so that the source_name can be extracted from Fluentd's buffer (default `source_name`)
-* `source_host` - Set _sourceHost metadata field within SumoLogic (default is `nil`)
+* `source_host`<sup>*</sup> - Set _sourceHost metadata field within SumoLogic (default is `nil`)
 * `log_format` - Format to post logs into Sumo. (default `json`)
   * text - Logs will appear in SumoLogic in text format (taken from the field specified in `log_key`)
   * json - Logs will appear in SumoLogic in json format.
@@ -37,6 +37,8 @@ Configuration options for fluent.conf are:
 * `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
 * `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
 
+__NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
+
 ### Example Configuration
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "1.4.1"
+  gem.version       = "1.5.0"
   gem.authors       = ["Steven Adams", "Frank Reno"]
   gem.email         = ["stevezau@gmail.com", "frank.reno@me.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}

data/lib/fluent/plugin/out_sumologic.rb

@@ -12,14 +12,14 @@ class SumologicConnection
     create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
   end
 
-  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type)
-    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type))
+  def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields)
+    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields))
     unless response.ok?
       raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
   end
 
-  def request_headers(source_host, source_category, source_name, data_type, metric_data_format)
+  def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields)
     headers = {
         'X-Sumo-Name'     => source_name,
         'X-Sumo-Category' => source_category,
@@ -38,6 +38,9 @@ class SumologicConnection
         raise RuntimeError, "Invalid #{metric_data_format}, must be graphite or carbon2 or prometheus"
       end
     end
+    unless collected_fields.nil?
+      headers['X-Sumo-Fields'] = collected_fields
+    end
     return headers
   end
 
@@ -114,8 +117,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
 
     if conf['data_type'].nil? || conf['data_type'] == LOGS_DATA_TYPE
       unless conf['log_format'].nil?
-        unless conf['log_format'] =~ /\A(?:json|text|json_merge)\z/
-          raise Fluent::ConfigError, "Invalid log_format #{conf['log_format']} must be text, json or json_merge"
+        unless conf['log_format'] =~ /\A(?:json|text|json_merge|fields)\z/
+          raise Fluent::ConfigError, "Invalid log_format #{conf['log_format']} must be text, json, json_merge or fields"
         end
       end
     end
@@ -200,9 +203,28 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
     time.to_s.length == 13 ? time : time * 1000
   end
 
+  def sumo_fields(sumo_metadata)
+    fields = sumo_metadata['fields'] || ""
+    Hash[
+        fields.split(',').map do |pair|
+          k, v = pair.split('=', 2)
+          [k, v]
+        end
+    ]
+  end
+
+  def dump_collected_fields(log_fields)
+    if log_fields.nil?
+      log_fields
+    else
+      log_fields.map{|k,v| "#{k}=#{v}"}.join(',')
+    end
+  end
+
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
     messages_list = {}
+    log_fields = nil
 
     # Sort messages
     chunk.msgpack_each do |time, record|
@@ -229,6 +251,12 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
             record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
           end
           log = dump_log(merge_json(record))
+        when 'fields'
+          log_fields = sumo_fields(sumo_metadata)
+          if @add_timestamp
+            record = {  @timestamp_key => sumo_timestamp(time) }.merge(record)
+          end
+          log = dump_log(record)
         else
           if @add_timestamp
             record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
@@ -261,7 +289,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
           source_category     =source_category,
           source_name         =source_name,
           data_type           =@data_type,
-          metric_data_format  =@metric_data_format
+          metric_data_format  =@metric_data_format,
+          collected_fields    =dump_collected_fields(log_fields)
       )
     end
 

data/test/plugin/test_out_sumologic.rb

@@ -37,7 +37,7 @@ class SumologicOutput < Test::Unit::TestCase
       log_format    foo
     }
     exception = assert_raise(Fluent::ConfigError) {create_driver(config)}
-    assert_equal("Invalid log_format foo must be text, json or json_merge", exception.message)
+    assert_equal("Invalid log_format foo must be text, json, json_merge or fields", exception.message)
   end
 
   def test_invalid_metrics_data_type
@@ -116,6 +116,27 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_empty_fields
+    config = %{
+	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+	      log_format      fields
+	      source_category test
+	      source_host     test
+	      source_name     test
+
+	    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    driver.run do
+      driver.feed("output.test", time, {'message' => 'test'})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
+                     body: /\A{"timestamp":\d+.,"message":"test"}\z/,
+                     times:1
+  end
+
   def test_emit_json_double_encoded
     config = %{
       endpoint        https://endpoint3.collection.us2.sumologic.com/receiver/v1/http/1234
@@ -200,6 +221,52 @@ class SumologicOutput < Test::Unit::TestCase
                      times:1
   end
 
+  def test_emit_with_sumo_metadata_with_fields_json_format
+    config = %{
+	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+	      log_format      json
+	    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'bar', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test'},
+                     body: /\A{"timestamp":\d+.,"foo":"bar","message":"test"}\z/,
+                     times:1
+  end
+
+  def test_emit_with_sumo_metadata_with_fields_fields_format
+    config = %{
+	      endpoint        https://collectors.sumologic.com/v1/receivers/http/1234
+	      log_format      fields
+	    }
+    driver = create_driver(config)
+    time = event_time
+    stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
+    ENV['HOST'] = "foo"
+    driver.run do
+      driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
+          "host": "#{ENV['HOST']}",
+          "source": "${tag}",
+          "category": "test",
+          "fields": "foo=bar, sumo = logic"
+      }})
+    end
+    assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
+                     headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic'},
+                     body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
+                     times:1
+  end
+
   def test_emit_with_sumo_metadata
     config = %{
       endpoint        https://collectors.sumologic.com/v1/receivers/http/1234

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Steven Adams
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-13 00:00:00.000000000 Z
+date: 2019-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: Output plugin to SumoLogic HTTP Endpoint