logstash-output-sumologic 1.2.0 → 1.3.2

data/lib/logstash/outputs/sumologic/compressor.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "stringio"
-require "zlib"
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class Compressor
 
+    require "stringio"
+    require "zlib"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     def initialize(config)

data/lib/logstash/outputs/sumologic/header_builder.rb

@@ -1,31 +1,17 @@
 # encoding: utf-8
-require "socket"
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class HeaderBuilder
 
+    require "socket"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
-    CONTENT_TYPE = "Content-Type"
-    CONTENT_TYPE_LOG = "text/plain"
-    CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
-    CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
-    CONTENT_ENCODING = "Content-Encoding"
-
-    CATEGORY_HEADER = "X-Sumo-Category"
-    CATEGORY_HEADER_DEFAULT = "Logstash"
-    HOST_HEADER = "X-Sumo-Host"
-    NAME_HEADER = "X-Sumo-Name"
-    NAME_HEADER_DEFAULT = "logstash-output-sumologic"
-
-    CLIENT_HEADER = "X-Sumo-Client"
-    CLIENT_HEADER_VALUE = "logstash-output-sumologic"
-
     def initialize(config)
       
       @extra_headers = config["extra_headers"] ||= {}
       @source_category = config["source_category"] ||= CATEGORY_HEADER_DEFAULT
+      @stats_category = config["stats_category"] ||= CATEGORY_HEADER_DEFAULT_STATS
       @source_host = config["source_host"] ||= Socket.gethostname
       @source_name = config["source_name"] ||= NAME_HEADER_DEFAULT
       @metrics = config["metrics"]
@@ -36,31 +22,31 @@ module LogStash; module Outputs; class SumoLogic;
 
     end # def initialize
     
-    def build()
-      headers = build_common()
-      headers[CATEGORY_HEADER] = @source_category unless @source_category.blank?
+    def build(event)
+      headers = Hash.new
+      headers.merge!(@extra_headers)
+      headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
+      headers[CATEGORY_HEADER] = event.sprintf(@source_category) unless blank?(@source_category)
+      headers[HOST_HEADER] = event.sprintf(@source_host) unless blank?(@source_host)
+      headers[NAME_HEADER] = event.sprintf(@source_name) unless blank?(@source_name)
       append_content_header(headers)
+      append_compress_header(headers)
       headers
     end # def build
 
     def build_stats()
-      headers = build_common()
-      headers[CATEGORY_HEADER] = "#{@source_category}.stats"
-      headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
-      headers
-    end # def build_stats
-
-    private
-    def build_common()
-      headers = Hash.new()
+      headers = Hash.new
       headers.merge!(@extra_headers)
       headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
-      headers[HOST_HEADER] = @source_host unless @source_host.blank?
-      headers[NAME_HEADER] = @source_name unless @source_name.blank?
+      headers[CATEGORY_HEADER] = @stats_category
+      headers[HOST_HEADER] = Socket.gethostname
+      headers[NAME_HEADER] = NAME_HEADER_DEFAULT
+      headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2 
       append_compress_header(headers)
       headers
-    end # build_common
+    end # def build_stats
 
+    private
     def append_content_header(headers)
       contentType = CONTENT_TYPE_LOG
       if @metrics || @fields_as_metrics
@@ -76,4 +62,4 @@ module LogStash; module Outputs; class SumoLogic;
     end # append_compress_header
 
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/sumologic/message_queue.rb

@@ -1,28 +1,43 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-
 module LogStash; module Outputs; class SumoLogic;
   class MessageQueue
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    include LogStash::Outputs::SumoLogic::Common
+
     def initialize(stats, config)
       @queue_max = (config["queue_max"] ||= 1) < 1 ? 1 : config["queue_max"]
       @queue = SizedQueue::new(@queue_max)
+      log_info("initialize memory queue", :max => @queue_max)
+      @queue_bytesize = Concurrent::AtomicFixnum.new
       @stats = stats
-    end
+    end # def initialize
 
-    def enq(obj)
-      if (obj.bytesize > 0)
-        @queue.enq(obj)
-        @stats.record_enque(obj)
-      end
-    end # def push
+    def enq(batch)
+      batch_size = batch.payload.bytesize
+      if (batch_size > 0)
+        @queue.enq(batch)
+        @stats.record_enque(batch_size)
+        @queue_bytesize.update { |v| v + batch_size }
+        log_dbg("enqueue",
+          :objects_in_queue => size,
+          :bytes_in_queue => @queue_bytesize,
+          :size => batch_size)
+        end
+    end # def enq
 
     def deq()
-      obj = @queue.deq()
-      @stats.record_deque(obj)
-      obj
-    end # def pop
+      batch = @queue.deq()
+      batch_size = batch.payload.bytesize
+      @stats.record_deque(batch_size)
+      @queue_bytesize.update { |v| v - batch_size }
+      log_dbg("dequeue",
+        :objects_in_queue => size,
+        :bytes_in_queue => @queue_bytesize,
+        :size => batch_size)
+      batch
+    end # def deq
 
     def drain()
       @queue.size.times.map {
@@ -33,6 +48,10 @@ module LogStash; module Outputs; class SumoLogic;
     def size()
       @queue.size()
     end # size
+
+    def bytesize()
+      @queue_bytesize.value
+    end # bytesize
     
   end
 end; end; end

data/lib/logstash/outputs/sumologic/monitor.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Monitor
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :is_pile
@@ -14,6 +14,7 @@ module LogStash; module Outputs; class SumoLogic;
       @queue = queue
       @stats = stats
       @stopping = Concurrent::AtomicBoolean.new(false)
+      @header_builder = HeaderBuilder.new(config)
 
       @enabled = config["stats_enabled"] ||= false
       @interval = config["stats_interval"] ||= 60
@@ -21,13 +22,14 @@ module LogStash; module Outputs; class SumoLogic;
     end # initialize
 
     def start()
+      log_info("starting monitor...", :interval => @interval)
       @stopping.make_false()
       if (@enabled)
         @monitor_t = Thread.new { 
           while @stopping.false?
             Stud.stoppable_sleep(@interval) { @stopping.true? }
-            if @stats.total_input_events.value > 0
-              @queue.enq(build_stats_payload())
+            if @stats.total_log_lines.value > 0 || @stats.total_metrics_datapoints.value > 0
+              @queue.enq(Batch.new(@header_builder.build_stats(), build_stats_payload()))
             end
           end # while
         }
@@ -37,9 +39,9 @@ module LogStash; module Outputs; class SumoLogic;
     def stop()
       @stopping.make_true()
       if (@enabled)
-        log_info "shutting down monitor..."
+        log_info("shutting down monitor...")
         @monitor_t.join
-        log_info "monitor is fully shutted down"
+        log_info("monitor is fully shutted down")
       end
     end # def stop
 
@@ -58,9 +60,12 @@ module LogStash; module Outputs; class SumoLogic;
         "total_response_success"
       ].map { |key|
         value = @stats.send(key).value
+        log_dbg("stats",
+          :key => key,
+          :value => value)
         build_metric_line(key, value, timestamp)
       }.join($/)
-      
+
       "#{STATS_TAG}#{counters}"
     end # def build_stats_payload
 
@@ -69,4 +74,4 @@ module LogStash; module Outputs; class SumoLogic;
     end # def build_metric_line
 
   end
-end; end; end        
+end; end; end

data/lib/logstash/outputs/sumologic/payload_builder.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
-require "logstash/json"
-require "logstash/event"
-
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class PayloadBuilder
     
+    require "logstash/json"
+    require "logstash/event"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     TIMESTAMP_FIELD = "@timestamp"
@@ -128,12 +127,17 @@ module LogStash; module Outputs; class SumoLogic;
     end # def expand_hash
     
     def apply_template(template, event)
-      if template.include? JSON_PLACEHOLDER
+      if template == JSON_PLACEHOLDER
+        hash = event2hash(event)
+        LogStash::Json.dump(hash)
+      elsif template.include? JSON_PLACEHOLDER
+        result = event.sprintf(template)
         hash = event2hash(event)
         dump = LogStash::Json.dump(hash)
-        template = template.gsub(JSON_PLACEHOLDER) { dump }
+        result.gsub(JSON_PLACEHOLDER) { dump }
+      else
+        event.sprintf(template)
       end
-      event.sprintf(template)
     end # def expand
     
     def get_metrics_name(event, name)

data/lib/logstash/outputs/sumologic/piler.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Piler
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :is_pile
@@ -17,23 +17,26 @@ module LogStash; module Outputs; class SumoLogic;
       @queue = queue
       @stats = stats
       @stopping = Concurrent::AtomicBoolean.new(false)
+      @payload_builder = PayloadBuilder.new(@stats, config)
+      @header_builder = HeaderBuilder.new(config)
       @is_pile = (@interval > 0 && @pile_max > 0)
-
       if (@is_pile)
-        @pile = Array.new
-        @pile_size = 0
+        @pile = Hash.new("")
         @semaphore = Mutex.new
       end
-
+  
     end # def initialize
 
     def start()
       @stopping.make_false()
       if (@is_pile)
+        log_info("starting piler...", 
+          :max => @pile_max, 
+          :timeout => @interval)
         @piler_t = Thread.new { 
           while @stopping.false?
             Stud.stoppable_sleep(@interval) { @stopping.true? }
-            log_dbg("timeout, enqueue pile now")
+            log_dbg("timeout", :timeout => @interval)
             enq_and_clear()
           end # while
         }
@@ -43,45 +46,44 @@ module LogStash; module Outputs; class SumoLogic;
     def stop()
       @stopping.make_true()
       if (@is_pile)
-        log_info "shutting down piler..."
-        @piler_t.join
-        log_info "piler is fully shutted down"
+        log_info("shutting down piler in #{@interval * 2} secs ...")
+        @piler_t.join(@interval * 2)
+        log_info("piler is fully shutted down")
       end
     end # def stop
 
-    def input(entry)
+    def input(event)
       if (@stopping.true?)
-        log_warn "piler is shutting down, message ignored", "message" => entry
-      elsif (@is_pile)
-        @semaphore.synchronize {
-          if @pile_size + entry.bytesize > @pile_max
-            @queue.enq(@pile.join($/))
-            @pile.clear
-            @pile_size = 0
-            @stats.record_clear_pile()
-          end
-          @pile << entry
-          @pile_size += entry.bytesize
-          @stats.record_input(entry)
-        }
+        log_warn("piler is shutting down, event is dropped", 
+          "event" => event)
       else
-        @queue.enq(entry)
-      end # if
+        headers = @header_builder.build(event)
+        payload = @payload_builder.build(event)
+        if (@is_pile)
+          @semaphore.synchronize {
+            content = @pile[headers]
+            size = content.bytesize
+            if size + payload.bytesize > @pile_max
+              @queue.enq(Batch.new(headers, content))
+              @pile[headers] = ""
+            end
+            @pile[headers] = blank?(@pile[headers]) ? payload : "#{@pile[headers]}\n#{payload}"
+          }
+        else
+          @queue.enq(Batch.new(headers, payload))
+        end # if
+      end
     end # def input
 
     private
     def enq_and_clear()
-      if (@pile.size > 0)
-        @semaphore.synchronize {
-          if (@pile.size > 0)
-            @queue.enq(@pile.join($/))
-            @pile.clear
-            @pile_size = 0
-            @stats.record_clear_pile()
-          end
-        }
-      end
+      @semaphore.synchronize {
+        @pile.each do |headers, content|
+          @queue.enq(Batch.new(headers, content))
+        end
+        @pile.clear()
+      }
     end # def enq_and_clear
 
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/sumologic/sender.rb

@@ -1,19 +1,19 @@
 # encoding: utf-8
-require "net/https"
-require "socket"
-require "thread"
-require "uri"
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/compressor"
-require "logstash/outputs/sumologic/header_builder"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Sender
 
+    require "net/https"
+    require "socket"
+    require "thread"
+    require "uri"
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/compressor"
+    require "logstash/outputs/sumologic/header_builder"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
-    STOP_TAG = "PLUGIN STOPPED"
+
 
     def initialize(client, queue, stats, config)
       @client = client
@@ -28,37 +28,37 @@ module LogStash; module Outputs; class SumoLogic;
       @tokens = SizedQueue.new(@sender_max)
       @sender_max.times { |t| @tokens << t }
 
-      @header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
-      @headers = @header_builder.build()
-      @stats_headers = @header_builder.build_stats()
       @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
 
     end # def initialize
 
     def start()
+      log_info("starting sender...",
+        :max => @sender_max, 
+        :requeue => @sleep_before_requeue)
       @stopping.make_false()
       @sender_t = Thread.new {
         while @stopping.false?
-          content = @queue.deq()
-          send_request(content)
+          batch = @queue.deq()
+          send_request(batch)
         end # while
-        @queue.drain().map { |content| 
-          send_request(content)
+        @queue.drain().map { |batch| 
+          send_request(batch)
         }
-        log_info "waiting messages sent out..."
+        log_info("waiting while senders finishing...")
         while @tokens.size < @sender_max
           sleep 1
         end # while
       }
-    end # def start_sender
+    end # def start
 
     def stop()
-      log_info "shutting down sender..."
+      log_info("shutting down sender...")
       @stopping.make_true()
-      @queue.enq(STOP_TAG)
+      @queue.enq(Batch.new(Hash.new, STOP_TAG))
       @sender_t.join
-      log_info "sender is fully shutted down"
-    end # def stop_sender
+      log_info("sender is fully shutted down")
+    end # def stop
     
     def connect()
       uri = URI.parse(@url)
@@ -68,34 +68,33 @@ module LogStash; module Outputs; class SumoLogic;
       begin
         res = http.request(request)
         if res.code.to_i != 200
-          log_err(
-            "Server rejected the request",
+          log_err("ping rejected",
             :url => @url,
-            :code => res.code
-          )
+            :code => res.code,
+            :body => res.body)
           false
         else
-          log_dbg(
-            "Server accepted the request",
-            :url => @url
-          )
+          log_info("ping accepted",
+            :url => @url)
           true
         end
-      rescue Exception => ex
-        log_err(
-          "Cannot connect to given url",
+      rescue Exception => exception
+        log_err("ping failed",
           :url => @url,
-          :exception => ex
-        )
+          :message => exception.message,
+          :class => exception.class.name,
+          :backtrace => exception.backtrace)
         false
       end
     end # def connect
     
     private
 
-    def send_request(content)
+    def send_request(batch)
+      content = batch.payload
+      headers = batch.headers
       if content == STOP_TAG
-        log_dbg "STOP_TAG is received."
+        log_info("STOP_TAG is received.")
         return
       end
       
@@ -103,12 +102,15 @@ module LogStash; module Outputs; class SumoLogic;
 
       if @stats_enabled && content.start_with?(STATS_TAG)
         body = @compressor.compress(content[STATS_TAG.length..-1])
-        headers = @stats_headers
       else
         body = @compressor.compress(content)
-        headers = @headers
       end
-  
+
+      log_dbg("sending request",
+        :headers => headers,
+        :content_size => content.size,
+        :content => content[0..20],
+        :payload_size => body.size)
       request = @client.send(:background).send(:post, @url, :body => body, :headers => headers)
       
       request.on_complete do
@@ -118,19 +120,16 @@ module LogStash; module Outputs; class SumoLogic;
       request.on_success do |response|
         @stats.record_response_success(response.code)
         if response.code < 200 || response.code > 299
-          log_err(
-            "HTTP request rejected(#{response.code})",
+          log_err("request rejected",
             :token => token,
             :code => response.code,
             :headers => headers,
-            :contet => content[0..20]
-          )
+            :contet => content[0..20])
           if response.code == 429 || response.code == 503 || response.code == 504
-            requeue_message(content)
+            requeue_message(batch)
           end
         else
-          log_dbg(
-            "HTTP request accepted",
+          log_dbg("request accepted",
             :token => token,
             :code => response.code)
         end
@@ -138,30 +137,34 @@ module LogStash; module Outputs; class SumoLogic;
   
       request.on_failure do |exception|
         @stats.record_response_failure()
-        log_err(
-          "Error in network transmission",
+        log_err("error in network transmission",
           :token => token,
           :message => exception.message,
           :class => exception.class.name,
-          :backtrace => exception.backtrace
-        )
-        requeue_message(content)
+          :backtrace => exception.backtrace)
+        requeue_message(batch)
       end      
 
       @stats.record_request(content.bytesize, body.bytesize)
       request.call
     end # def send_request
 
-    def requeue_message(content)
-      if @stopping.false? && @sleep_before_requeue >= 0
-        log_warn(
-          "requeue message",
+    def requeue_message(batch)
+      content = batch.payload
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        log_warn("do not requeue stats payload",
+          :content => content)
+      elsif @stopping.false? && @sleep_before_requeue >= 0
+        log_info("requeue message",
           :after => @sleep_before_requeue,
-          :content => content[0..20])
+          :queue_size => @queue.size,
+          :content_size => content.size,
+          :content => content[0..20],
+          :headers => batch.headers)
         Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
-        @queue.enq(content)
+        @queue.enq(batch)
       end
     end # def reque_message
 
   end
-end; end; end
+end; end; end