logstash-output-sumologic 1.2.2 → 1.4.0

data/lib/logstash/outputs/sumologic/sender.rb

@@ -1,19 +1,19 @@
 # encoding: utf-8
-require "net/https"
-require "socket"
-require "thread"
-require "uri"
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/compressor"
-require "logstash/outputs/sumologic/header_builder"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Sender
 
+    require "net/https"
+    require "socket"
+    require "thread"
+    require "uri"
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/compressor"
+    require "logstash/outputs/sumologic/header_builder"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
-    STOP_TAG = "PLUGIN STOPPED"
+
 
     def initialize(client, queue, stats, config)
       @client = client
@@ -28,9 +28,6 @@ module LogStash; module Outputs; class SumoLogic;
       @tokens = SizedQueue.new(@sender_max)
       @sender_max.times { |t| @tokens << t }
 
-      @header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
-      @headers = @header_builder.build()
-      @stats_headers = @header_builder.build_stats()
       @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
 
     end # def initialize
@@ -42,11 +39,11 @@ module LogStash; module Outputs; class SumoLogic;
       @stopping.make_false()
       @sender_t = Thread.new {
         while @stopping.false?
-          content = @queue.deq()
-          send_request(content)
+          batch = @queue.deq()
+          send_request(batch)
         end # while
-        @queue.drain().map { |content| 
-          send_request(content)
+        @queue.drain().map { |batch| 
+          send_request(batch)
         }
         log_info("waiting while senders finishing...")
         while @tokens.size < @sender_max
@@ -58,7 +55,7 @@ module LogStash; module Outputs; class SumoLogic;
     def stop()
       log_info("shutting down sender...")
       @stopping.make_true()
-      @queue.enq(STOP_TAG)
+      @queue.enq(Batch.new(Hash.new, STOP_TAG))
       @sender_t.join
       log_info("sender is fully shutted down")
     end # def stop
@@ -93,7 +90,9 @@ module LogStash; module Outputs; class SumoLogic;
     
     private
 
-    def send_request(content)
+    def send_request(batch)
+      content = batch.payload
+      headers = batch.headers
       if content == STOP_TAG
         log_info("STOP_TAG is received.")
         return
@@ -103,10 +102,8 @@ module LogStash; module Outputs; class SumoLogic;
 
       if @stats_enabled && content.start_with?(STATS_TAG)
         body = @compressor.compress(content[STATS_TAG.length..-1])
-        headers = @stats_headers
       else
         body = @compressor.compress(content)
-        headers = @headers
       end
 
       log_dbg("sending request",
@@ -128,8 +125,8 @@ module LogStash; module Outputs; class SumoLogic;
             :code => response.code,
             :headers => headers,
             :contet => content[0..20])
-          if response.code == 429 || response.code == 503 || response.code == 504
-            requeue_message(content)
+          if response.code == 429 || response.code == 502 || response.code == 503 || response.code == 504
+            requeue_message(batch)
           end
         else
           log_dbg("request accepted",
@@ -145,24 +142,29 @@ module LogStash; module Outputs; class SumoLogic;
           :message => exception.message,
           :class => exception.class.name,
           :backtrace => exception.backtrace)
-        requeue_message(content)
+        requeue_message(batch)
       end      
 
       @stats.record_request(content.bytesize, body.bytesize)
       request.call
     end # def send_request
 
-    def requeue_message(content)
-      if @stopping.false? && @sleep_before_requeue >= 0
+    def requeue_message(batch)
+      content = batch.payload
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        log_warn("do not requeue stats payload",
+          :content => content)
+      elsif @stopping.false? && @sleep_before_requeue >= 0
         log_info("requeue message",
           :after => @sleep_before_requeue,
           :queue_size => @queue.size,
           :content_size => content.size,
-          :content => content[0..20])
+          :content => content[0..20],
+          :headers => batch.headers)
         Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
-        @queue.enq(content)
+        @queue.enq(batch)
       end
     end # def reque_message
 
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/sumologic/statistics.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class Statistics
 
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :initialize_time
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
     attr_reader :total_input_bytes
     attr_reader :total_metrics_datapoints
     attr_reader :total_log_lines
-    attr_reader :current_pile_items
-    attr_reader :current_pile_bytes
     attr_reader :total_enque_times
     attr_reader :total_enque_bytes
     attr_reader :total_deque_times
     attr_reader :total_deque_bytes
-    attr_reader :current_queue_items
-    attr_reader :current_queue_bytes
     attr_reader :total_output_requests
     attr_reader :total_output_bytes
     attr_reader :total_output_bytes_compressed 
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
       @total_input_bytes = Concurrent::AtomicFixnum.new
       @total_metrics_datapoints = Concurrent::AtomicFixnum.new
       @total_log_lines = Concurrent::AtomicFixnum.new
-      @current_pile_items = Concurrent::AtomicFixnum.new
-      @current_pile_bytes = Concurrent::AtomicFixnum.new
       @total_enque_times = Concurrent::AtomicFixnum.new
       @total_enque_bytes = Concurrent::AtomicFixnum.new
       @total_deque_times = Concurrent::AtomicFixnum.new
       @total_deque_bytes = Concurrent::AtomicFixnum.new
-      @current_queue_items = Concurrent::AtomicFixnum.new
-      @current_queue_bytes = Concurrent::AtomicFixnum.new
       @total_output_requests = Concurrent::AtomicFixnum.new
       @total_output_bytes = Concurrent::AtomicFixnum.new
       @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
       @total_response.get(key) ? @total_response.get(key).value : 0
     end
 
-    def record_multi_input(events, bytesize)
-      @total_input_events.update { |v| v + events }
-      @total_input_bytes.update { |v| v + bytesize }
-    end # def record_multi_input
-
-    def record_input(entry)
+    def record_input(size)
       @total_input_events.increment()
-      @total_input_bytes.update { |v| v + entry.bytesize }
-      @current_pile_items.increment()
-      @current_pile_bytes.update { |v| v + entry.bytesize }
+      @total_input_bytes.update { |v| v + size }
     end # def record_input
 
     def record_log_process()
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
       @total_metrics_datapoints.update { |v| v + dps }
     end # def record_metrics_process
 
-    def record_clear_pile()
-      @current_pile_items.value= 0
-      @current_pile_bytes.value= 0
-    end # def record_pile_clear
-
-    def record_enque(payload)
+    def record_enque(size)
       @total_enque_times.increment()
-      @total_enque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.increment()
-      @current_queue_bytes.update { |v| v + payload.bytesize }
+      @total_enque_bytes.update { |v| v + size }
     end # def record_enque
 
-    def record_deque(payload)
+    def record_deque(size)
       @total_deque_times.increment()
-      @total_deque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.decrement()
-      @current_queue_bytes.update { |v| v - payload.bytesize }
+      @total_deque_bytes.update { |v| v + size }
     end # def record_deque
 
     def record_request(size, size_compressed)

data/lib/logstash/outputs/sumologic.rb

@@ -14,6 +14,7 @@ require "logstash/plugin_mixins/http_client"
 class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   declare_threadsafe!
   
+  require "logstash/outputs/sumologic/batch"
   require "logstash/outputs/sumologic/common"
   require "logstash/outputs/sumologic/compressor"
   require "logstash/outputs/sumologic/header_builder"
@@ -101,22 +102,26 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   # For carbon2 metrics format only, define the meta tags (which will NOT be used to identify the metrics)
   config :meta_tags, :validate => :hash, :default => {}
   
-  # For messages fail to send or get 429/503/504 response, try to resend after (x) seconds; don't resend if (x) < 0
+  # For messages fail to send or get 429/502/503/504 response, try to resend after (x) seconds; don't resend if (x) < 0
   config :sleep_before_requeue, :validate => :number, :default => 30
 
+  config :stats_category, :validate => :string, :default => CATEGORY_HEADER_DEFAULT_STATS
+
   # Sending throughput data as metrics
   config :stats_enabled, :validate => :boolean, :default => false
 
   # Sending throughput data points every (x) seconds
   config :stats_interval, :validate => :number, :default => 60
 
+  # Disable cookies by default (used in HTTP mixin)
+  config :cookies, :validate => :boolean, :default => false
+
   attr_reader :stats
   
   def register
     set_logger(@logger)
-    @stats = Statistics.new()
+    @stats = Statistics.new
     @queue = MessageQueue.new(@stats, config)
-    @builder = PayloadBuilder.new(@stats, config)
     @piler = Piler.new(@queue, @stats, config)
     @monitor = Monitor.new(@queue, @stats, config)
     @sender = Sender.new(client, @queue, @stats, config)
@@ -130,24 +135,12 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   end # def register
 
   def multi_receive(events)
-    # events.map { |e| receive(e) }
-    begin
-      content = Array(events).map { |event| @builder.build(event) }.join($/)
-      @queue.enq(content)
-      @stats.record_multi_input(events.size, content.bytesize)
-    rescue Exception => exception
-      log_err("error when processing events",
-        :events => events,
-        :message => exception.message,
-        :class => exception.class.name,
-        :backtrace => exception.backtrace)
-    end
+    Array(events).map { |event| receive(event) }
   end # def multi_receive
   
   def receive(event)
     begin
-      content = @builder.build(event)
-      @piler.input(content)
+      @piler.input(event)
     rescue Exception => exception
       log_err("error when processing event",
         :event => event,

data/logstash-output-sumologic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sumologic'
-  s.version       = '1.2.2'
+  s.version       = '1.4.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Deliever the log to Sumo Logic cloud service.'
   s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-mixin-http_client'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '>= 6', '< 8'
   
+  s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
 describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   result = {}
+  event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
 
   before :each do
-    result = builder.build()
+    result = builder.build(event)
   end
 
   context "should build headers by default" do
+
     let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
 
     specify {
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_category with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category bar")
+    }
+
+  end # context
+
   context "should override source_name" do
     
     let(:builder) {
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_name with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name bar")
+    }
+
+  end # context
+
   context "should override source_host" do
     
     let(:builder) {
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_host with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host bar")
+    }
+
+  end # context
+
   context "should hornor extra_headers" do
     
     let(:builder) {
@@ -194,4 +241,90 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should build headers for stats" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expected = {
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash.stats",
+        "Content-Type" => "application/vnd.sumologic.carbon2"
+      }
+      expect(stats_result).to eq(expected)
+    }
+  end
+
+  context "should build headers for stats with overridden source category" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "stats_category" => "custom")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result["X-Sumo-Category"]).to eq("custom")
+    }
+  end
+
+  context "should build headers for stats with compression" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true, "compress_encoding" => "gzip")
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result["Content-Encoding"]).to eq("gzip")
+    }
+  end
+
+  context "should build headers for stats with extra_headers" do
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar"
+        })
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expect(stats_result.count).to eq(6)
+      expect(stats_result["foo"]).to eq("bar")
+    }
+  end
+
+  context "should build headers for stats with extra_headers but never overwrite pre-defined headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar",
+          "X-Sumo-Client" => "a",
+          "X-Sumo-Name" => "b",
+          "X-Sumo-Host" => "c",
+          "X-Sumo-Category" => "d",
+          "Content-Type" => "e"
+      })
+    }
+
+    specify {
+      stats_result = builder.build_stats()
+      expected = {
+        "foo" => "bar",
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash.stats",
+        "Content-Type" => "application/vnd.sumologic.carbon2"
+      }
+      expect(stats_result).to eq(expected)
+    }
+
+  end # context
+
 end # describe

data/spec/outputs/sumologic/message_queue_spec.rb

@@ -1,46 +1,48 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sumologic"
+include LogStash::Outputs
+
+describe SumoLogic::MessageQueue do
 
-describe LogStash::Outputs::SumoLogic::MessageQueue do
-  
   context "working in pile mode if interval > 0 && pile_max > 0" do
 
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
+    let(:stats) { SumoLogic::Statistics.new }
 
     it "enq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
         expect(queue.size()).to eq(i + 1)
         expect(stats.total_enque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(100)
     end
 
     it "deq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       10.times { |i|
         expect(queue.size()).to eq(10 - i)
         result = queue.deq()
-        expect(result).to eq("test - #{i}")
+        expect(result.payload).to eq("test  -  #{i}")
         expect(stats.total_deque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(0)
     end
 
     it "drain() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       result = queue.drain()
       expect(queue.size()).to eq(0)
       expect(stats.total_deque_times.value).to eq(10)
       expect(result.size).to eq(10)
+      expect(queue.bytesize()).to eq(0)
       10.times { |i|
-        expect(result[i]).to eq("test - #{i}")
+        expect(result[i].payload).to eq("test  -  #{i}")
       }
     end