logstash-output-sumologic 1.3.1 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 596877e694163207291385444d3e10ced75cba12422d57b21503126395002f03
-  data.tar.gz: 157f6fc6262ef8990d44d597066bbbcde226e27dc3d48d332a29970d38bd962c
+  metadata.gz: 0cb3e69cc8d33b60f981ee807b8deeab442bca032d5715df432d467d5a917b51
+  data.tar.gz: 4e1c485db70cace53f2c46b7fd089be90775e74d569ace0fd1462a98ffe7fed7
 SHA512:
-  metadata.gz: 2da6958efa3878652965d1620206a3af61172e1f5ff3f369ed738a0346c0c14949cad1c5e36c4d889e8e3dd75373cf8a6f6f0d6be0c8800dfda14b6be855f74b
-  data.tar.gz: b6021badd961cbe617108676906a55e16beeaa169e9f1a3e8a0b1244508c8271e661f7df8c951b38b7132d61588631a00f9595573abf50de43d69ce4dbcb6693
+  metadata.gz: '0478381fc42c308038c921dcede8662f5c83147f618c34bf5acf1f9c07b693d3a3e95a0978f74dc8114217b7e1c8b0aff78e29feb2435b27d6b1618a116fddb1'
+  data.tar.gz: f8ba1e23876443e518d468f1f2c162f4053093fa251f870b0abd61528caea9c6fdef88dd86ba91278519ee7fd5baf352479e9d97386c8e9cbcf817b19c41e2c9

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## 1.3.2 (2021-02-03)
+
+- [#60](https://github.com/SumoLogic/logstash-output-sumologic/pull/60) Fix plugin metrics not being sent
+
 ## 1.3.1 (2020-12-18)
 
 - [#53](https://github.com/SumoLogic/logstash-output-sumologic/pull/53) Fix "undefined method `blank?'"

data/DEVELOPER.md

@@ -35,8 +35,7 @@ docker run --rm -it -e 'sumo_url=https://events.sumologic.net/receiver/v1/http/X
 
 ### Running on bare metal
 
-The test requires JRuby to run. So you need to install [JRuby](http://jruby.org/), [bundle](https://bundler.io/bundle_install.html) and [RVM](https://rvm.io/) (for switching between JRuby and Ruby) first.
-And then run:
+The test requires JRuby to run.
 
 ```bash
 rvm use jruby
@@ -45,5 +44,12 @@ export sumo_url=https://events.sumologic.net/receiver/v1/http/XXXXXXXXXX
 rspec spec/
 ```
 
-The project is integrated to Travis CI now.
-Make sure [all test passed](https://travis-ci.com/SumoLogic/logstash-output-sumologic) before creating PR
+## Continuous Integration
+
+The project uses GitHub Actions for:
+
+- Testing pull requests and main branch commits: [.github/workflows/ci.yml](.github/workflows/ci.yml)
+- Publishing new version of gem to RubyGems.org after tagging: [.github/workflows/publish.yml](.github/workflows/publish.yml)
+
+Before publishing a new version, make sure the RubyGems account has MFA disabled for API access.
+Go to [Settings](https://rubygems.org/settings/edit) and set `MFA Level` to `UI only`.

data/README.md

@@ -112,6 +112,7 @@ Logon to Sumo Logic [web app](https://service.sumologic.com/) and run
 | `fields_include`       | array   | No        | all fields    | Working with `fields_as_metrics` parameter, only the fields which full name matching these RegEx pattern(s) will be included in metrics.
 | `fields_exclude`       | array   | No        | none          | Working with `fields_as_metrics` parameter, the fields which full name matching these RegEx pattern(s) will be ignored in metrics.
 | `sleep_before_requeue` | number  | No        | `30`          | The message failed to send to server will be retried after (x) seconds. Not retried if negative number set
+| `stats_category`       | string  | No        | `Logstash.stats` | The source category this plugin's stats will be sent with when `stats_enabled` is `true`
 | `stats_enabled`        | boolean | No        | `false`       | If `true`, stats of this plugin will be sent as metrics
 | `stats_interval`       | number  | No        | `60`          | The stats will be sent every (x) seconds
 
@@ -136,7 +137,7 @@ On the other side, this version is marked as thread safe so if necessary, multip
 
 ### Monitor throughput in metrics
 
-If your Sumo Logic account supports metrics feature, you can enable the stats monitor of this plugin with configuring `stats_enabled` to `true`. For every `stats_interval` seconds, a batch of metrics data points will be sent to Sumo Logic with source category `XXX.stats` (`XXX` is the source category of main output):
+If your Sumo Logic account supports metrics feature, you can enable the stats monitor of this plugin with configuring `stats_enabled` to `true`. For every `stats_interval` seconds, a batch of metrics data points will be sent to Sumo Logic with source category specified by `stats_category` parameter:
 
 | Metric                          | Description                                                 |
 | ------------------------------- | ----------------------------------------------------------- |

data/lib/logstash/outputs/sumologic.rb

@@ -105,6 +105,8 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   # For messages fail to send or get 429/503/504 response, try to resend after (x) seconds; don't resend if (x) < 0
   config :sleep_before_requeue, :validate => :number, :default => 30
 
+  config :stats_category, :validate => :string, :default => CATEGORY_HEADER_DEFAULT_STATS
+
   # Sending throughput data as metrics
   config :stats_enabled, :validate => :boolean, :default => false
 

data/lib/logstash/outputs/sumologic/common.rb

@@ -22,6 +22,7 @@ module LogStash; module Outputs; class SumoLogic;
 
     CATEGORY_HEADER = "X-Sumo-Category"
     CATEGORY_HEADER_DEFAULT = "Logstash"
+    CATEGORY_HEADER_DEFAULT_STATS = "Logstash.stats"
     HOST_HEADER = "X-Sumo-Host"
     NAME_HEADER = "X-Sumo-Name"
     NAME_HEADER_DEFAULT = "logstash-output-sumologic"

data/lib/logstash/outputs/sumologic/header_builder.rb

@@ -11,6 +11,7 @@ module LogStash; module Outputs; class SumoLogic;
       
       @extra_headers = config["extra_headers"] ||= {}
       @source_category = config["source_category"] ||= CATEGORY_HEADER_DEFAULT
+      @stats_category = config["stats_category"] ||= CATEGORY_HEADER_DEFAULT_STATS
       @source_host = config["source_host"] ||= Socket.gethostname
       @source_name = config["source_name"] ||= NAME_HEADER_DEFAULT
       @metrics = config["metrics"]
@@ -33,6 +34,18 @@ module LogStash; module Outputs; class SumoLogic;
       headers
     end # def build
 
+    def build_stats()
+      headers = Hash.new
+      headers.merge!(@extra_headers)
+      headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
+      headers[CATEGORY_HEADER] = @stats_category
+      headers[HOST_HEADER] = Socket.gethostname
+      headers[NAME_HEADER] = NAME_HEADER_DEFAULT
+      headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2 
+      append_compress_header(headers)
+      headers
+    end # def build_stats
+
     private
     def append_content_header(headers)
       contentType = CONTENT_TYPE_LOG

data/lib/logstash/outputs/sumologic/monitor.rb

@@ -14,6 +14,7 @@ module LogStash; module Outputs; class SumoLogic;
       @queue = queue
       @stats = stats
       @stopping = Concurrent::AtomicBoolean.new(false)
+      @header_builder = HeaderBuilder.new(config)
 
       @enabled = config["stats_enabled"] ||= false
       @interval = config["stats_interval"] ||= 60
@@ -27,8 +28,8 @@ module LogStash; module Outputs; class SumoLogic;
         @monitor_t = Thread.new { 
           while @stopping.false?
             Stud.stoppable_sleep(@interval) { @stopping.true? }
-            if @stats.total_input_events.value > 0
-              @queue.enq(build_stats_payload())
+            if @stats.total_log_lines.value > 0 || @stats.total_metrics_datapoints.value > 0
+              @queue.enq(Batch.new(@header_builder.build_stats(), build_stats_payload()))
             end
           end # while
         }
@@ -73,4 +74,4 @@ module LogStash; module Outputs; class SumoLogic;
     end # def build_metric_line
 
   end
-end; end; end        
+end; end; end

data/lib/logstash/outputs/sumologic/sender.rb

@@ -102,8 +102,6 @@ module LogStash; module Outputs; class SumoLogic;
 
       if @stats_enabled && content.start_with?(STATS_TAG)
         body = @compressor.compress(content[STATS_TAG.length..-1])
-        headers[CATEGORY_HEADER] = "#{headers[CATEGORY_HEADER]}.stats"
-        headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
       else
         body = @compressor.compress(content)
       end
@@ -169,4 +167,4 @@ module LogStash; module Outputs; class SumoLogic;
     end # def reque_message
 
   end
-end; end; end
+end; end; end

data/logstash-output-sumologic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sumologic'
-  s.version       = '1.3.1'
+  s.version       = '1.3.2'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Deliever the log to Sumo Logic cloud service.'
   s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -241,4 +241,90 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should build headers for stats" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expected = {
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash.stats",
+        "Content-Type" => "application/vnd.sumologic.carbon2"
+      }
+      expect(stats_result).to eq(expected)
+    }
+  end
+
+  context "should build headers for stats with overridden source category" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "stats_category" => "custom")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result["X-Sumo-Category"]).to eq("custom")
+    }
+  end
+
+  context "should build headers for stats with compression" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true, "compress_encoding" => "gzip")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result["Content-Encoding"]).to eq("gzip")
+    }
+  end
+
+  context "should build headers for stats with extra_headers" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar"
+        })
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result.count).to eq(6)
+      expect(stats_result["foo"]).to eq("bar")
+    }
+  end
+
+  context "should build headers for stats with extra_headers but never overwrite pre-defined headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar",
+          "X-Sumo-Client" => "a",
+          "X-Sumo-Name" => "b",
+          "X-Sumo-Host" => "c",
+          "X-Sumo-Category" => "d",
+          "Content-Type" => "e"
+      })
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expected = {
+        "foo" => "bar",
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash.stats",
+        "Content-Type" => "application/vnd.sumologic.carbon2"
+      }
+      expect(stats_result).to eq(expected)
+    }
+
+  end # context
+
 end # describe

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sumologic
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.2
 platform: ruby
 authors:
 - Sumo Logic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-18 00:00:00.000000000 Z
+date: 2021-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore