logstash-output-sumologic 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6f148933e50f3fad54125d1e006def5f8515add3a030d4e163aca083e6d0c6d
-  data.tar.gz: f1bcdaa22d833c71ac6b6779993427afd500d798417f7f33a498f150367b2f47
+  metadata.gz: 246eb9c67732ee8ec198f775b9dfa1bca652105c9a7c9b10504a3f3476a4468f
+  data.tar.gz: 06b46d47f19665b15a0b4404b8dc332c9ad29874520767353be3daddbb2920d4
 SHA512:
-  metadata.gz: 580d1c927da3234976e7c71beea83dcffe5e8dabb2e495f50197f91b8ac156e57740d33433c13a985d332542b2ae38613a3e3167adbe23c3ee3ad4ca2a91452f
-  data.tar.gz: be8464e519361f94ee835c93440ebfc9cb1b169b81cc62a477118753ec048a3742896a112fd82cc9123a8b864be756e7ec1114fcc725b5fa03bee370ba13248f
+  metadata.gz: 23bf3e2632570fe4c352f461a7b144156267b394dfa095c5c909e8376b28f067e2d0a3bcb12460e2af23fd532544f3cbc425451d9a4ee93de9cdcb806449bb5e
+  data.tar.gz: 31dad1b3410d87e45e220b39dec13129e2ac7e7b19281697947ee273a72e76d7bae792acdfaaaaa5a442bb94a47adba8edfafccff3d0a20a537caec6d7fa035e

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change Log
 
+## 1.3.0
+- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) Provide Docker image with Logstash 6.6 + output plugin on docker hub
+- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) Kubernetes support with Logstash beats to SumoLogic
+- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) CI improving 
+- [#36](https://github.com/SumoLogic/logstash-output-sumologic/pull/36) Update sender to send in batch.
+- [#36](https://github.com/SumoLogic/logstash-output-sumologic/pull/36) Support %{} field evaluation in `source_category`, `source_name`, `source_host` parameters
+- [#39](https://github.com/SumoLogic/logstash-output-sumologic/pull/39) Disable cookies by default
+
 ## 1.2.2
 
 - Bug fix: memory leak when using `%{@json}` in format
@@ -21,4 +29,4 @@
 
 ## 1.0.0
 
-- Initial release
+- Initial release

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Sumo Logic Output Plugin
 
-[![Build Status](https://travis-ci.org/SumoLogic/logstash-output-sumologic.svg?branch=master)](https://travis-ci.org/SumoLogic/logstash-output-sumologic)  [![Gem Version](https://badge.fury.io/rb/logstash-output-sumologic.svg)](https://badge.fury.io/rb/logstash-output-sumologic)
+[![Build Status](https://travis-ci.org/SumoLogic/logstash-output-sumologic.svg?branch=master)](https://travis-ci.org/SumoLogic/logstash-output-sumologic)  [![Gem Version](https://badge.fury.io/rb/logstash-output-sumologic.svg)](https://badge.fury.io/rb/logstash-output-sumologic)  [![Docker Pulls](https://img.shields.io/docker/pulls/sumologic/logstash-output-sumologic.svg)](https://hub.docker.com/r/sumologic/logstash-output-sumologic)
 
 This is an output plugin for [Logstash](https://github.com/elastic/logstash).
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/sumologic.rb

@@ -14,6 +14,7 @@ require "logstash/plugin_mixins/http_client"
 class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   declare_threadsafe!
   
+  require "logstash/outputs/sumologic/batch"
   require "logstash/outputs/sumologic/common"
   require "logstash/outputs/sumologic/compressor"
   require "logstash/outputs/sumologic/header_builder"
@@ -110,13 +111,15 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   # Sending throughput data points every (x) seconds
   config :stats_interval, :validate => :number, :default => 60
 
+  # Disable cookies by default (used in HTTP mixin)
+  config :cookies, :validate => :boolean, :default => false
+
   attr_reader :stats
   
   def register
     set_logger(@logger)
-    @stats = Statistics.new()
+    @stats = Statistics.new
     @queue = MessageQueue.new(@stats, config)
-    @builder = PayloadBuilder.new(@stats, config)
     @piler = Piler.new(@queue, @stats, config)
     @monitor = Monitor.new(@queue, @stats, config)
     @sender = Sender.new(client, @queue, @stats, config)
@@ -130,24 +133,12 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
   end # def register
 
   def multi_receive(events)
-    # events.map { |e| receive(e) }
-    begin
-      content = Array(events).map { |event| @builder.build(event) }.join($/)
-      @queue.enq(content)
-      @stats.record_multi_input(events.size, content.bytesize)
-    rescue Exception => exception
-      log_err("error when processing events",
-        :events => events,
-        :message => exception.message,
-        :class => exception.class.name,
-        :backtrace => exception.backtrace)
-    end
+    Array(events).map { |event| receive(event) }
   end # def multi_receive
   
   def receive(event)
     begin
-      content = @builder.build(event)
-      @piler.input(content)
+      @piler.input(event)
     rescue Exception => exception
       log_err("error when processing event",
         :event => event,

data/lib/logstash/outputs/sumologic/batch.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Batch
+    
+    attr_accessor :headers, :payload
+  
+    def initialize(headers, payload)
+        @headers, @payload = headers, payload
+    end
+  
+  end
+end; end; end;

data/lib/logstash/outputs/sumologic/common.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
-require "date"
-
 module LogStash; module Outputs; class SumoLogic;
   module Common
 
+    require "date"
+
     # global constants
     DEFAULT_LOG_FORMAT = "%{@timestamp} %{host} %{message}"
     METRICS_NAME_PLACEHOLDER = "*"
@@ -12,6 +12,22 @@ module LogStash; module Outputs; class SumoLogic;
     DEFLATE = "deflate"
     GZIP = "gzip"
     STATS_TAG = "STATS_TAG"
+    STOP_TAG = "PLUGIN STOPPED"
+
+    CONTENT_TYPE = "Content-Type"
+    CONTENT_TYPE_LOG = "text/plain"
+    CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
+    CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
+    CONTENT_ENCODING = "Content-Encoding"
+
+    CATEGORY_HEADER = "X-Sumo-Category"
+    CATEGORY_HEADER_DEFAULT = "Logstash"
+    HOST_HEADER = "X-Sumo-Host"
+    NAME_HEADER = "X-Sumo-Name"
+    NAME_HEADER_DEFAULT = "logstash-output-sumologic"
+
+    CLIENT_HEADER = "X-Sumo-Client"
+    CLIENT_HEADER_VALUE = "logstash-output-sumologic"
 
     # for debugging test
     LOG_TO_CONSOLE = false

data/lib/logstash/outputs/sumologic/compressor.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "stringio"
-require "zlib"
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class Compressor
 
+    require "stringio"
+    require "zlib"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     def initialize(config)

data/lib/logstash/outputs/sumologic/header_builder.rb

@@ -1,27 +1,12 @@
 # encoding: utf-8
-require "socket"
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class HeaderBuilder
 
+    require "socket"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
-    CONTENT_TYPE = "Content-Type"
-    CONTENT_TYPE_LOG = "text/plain"
-    CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
-    CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
-    CONTENT_ENCODING = "Content-Encoding"
-
-    CATEGORY_HEADER = "X-Sumo-Category"
-    CATEGORY_HEADER_DEFAULT = "Logstash"
-    HOST_HEADER = "X-Sumo-Host"
-    NAME_HEADER = "X-Sumo-Name"
-    NAME_HEADER_DEFAULT = "logstash-output-sumologic"
-
-    CLIENT_HEADER = "X-Sumo-Client"
-    CLIENT_HEADER_VALUE = "logstash-output-sumologic"
-
     def initialize(config)
       
       @extra_headers = config["extra_headers"] ||= {}
@@ -36,31 +21,19 @@ module LogStash; module Outputs; class SumoLogic;
 
     end # def initialize
     
-    def build()
-      headers = build_common()
-      headers[CATEGORY_HEADER] = @source_category unless @source_category.blank?
-      append_content_header(headers)
-      headers
-    end # def build
-
-    def build_stats()
-      headers = build_common()
-      headers[CATEGORY_HEADER] = "#{@source_category}.stats"
-      headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
-      headers
-    end # def build_stats
-
-    private
-    def build_common()
-      headers = Hash.new()
+    def build(event)
+      headers = Hash.new
       headers.merge!(@extra_headers)
       headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
-      headers[HOST_HEADER] = @source_host unless @source_host.blank?
-      headers[NAME_HEADER] = @source_name unless @source_name.blank?
+      headers[CATEGORY_HEADER] = event.sprintf(@source_category) unless @source_category.blank?
+      headers[HOST_HEADER] = event.sprintf(@source_host) unless @source_host.blank?
+      headers[NAME_HEADER] = event.sprintf(@source_name) unless @source_name.blank?
+      append_content_header(headers)
       append_compress_header(headers)
       headers
-    end # build_common
+    end # def build
 
+    private
     def append_content_header(headers)
       contentType = CONTENT_TYPE_LOG
       if @metrics || @fields_as_metrics

data/lib/logstash/outputs/sumologic/message_queue.rb

@@ -1,38 +1,42 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-
 module LogStash; module Outputs; class SumoLogic;
   class MessageQueue
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
     include LogStash::Outputs::SumoLogic::Common
 
     def initialize(stats, config)
       @queue_max = (config["queue_max"] ||= 1) < 1 ? 1 : config["queue_max"]
       @queue = SizedQueue::new(@queue_max)
       log_info("initialize memory queue", :max => @queue_max)
+      @queue_bytesize = Concurrent::AtomicFixnum.new
       @stats = stats
     end # def initialize
 
-    def enq(obj)
-      if (obj.bytesize > 0)
-        @queue.enq(obj)
-        @stats.record_enque(obj)
+    def enq(batch)
+      batch_size = batch.payload.bytesize
+      if (batch_size > 0)
+        @queue.enq(batch)
+        @stats.record_enque(batch_size)
+        @queue_bytesize.update { |v| v + batch_size }
         log_dbg("enqueue",
           :objects_in_queue => size,
-          :bytes_in_queue => @stats.current_queue_bytes,
-          :size => obj.bytesize)
-      end
+          :bytes_in_queue => @queue_bytesize,
+          :size => batch_size)
+        end
     end # def enq
 
     def deq()
-      obj = @queue.deq()
-      @stats.record_deque(obj)
+      batch = @queue.deq()
+      batch_size = batch.payload.bytesize
+      @stats.record_deque(batch_size)
+      @queue_bytesize.update { |v| v - batch_size }
       log_dbg("dequeue",
         :objects_in_queue => size,
-        :bytes_in_queue => @stats.current_queue_bytes,
-        :size => obj.bytesize)
-      obj
+        :bytes_in_queue => @queue_bytesize,
+        :size => batch_size)
+      batch
     end # def deq
 
     def drain()
@@ -44,6 +48,10 @@ module LogStash; module Outputs; class SumoLogic;
     def size()
       @queue.size()
     end # size
+
+    def bytesize()
+      @queue_bytesize.value
+    end # bytesize
     
   end
 end; end; end

data/lib/logstash/outputs/sumologic/monitor.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Monitor
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :is_pile

data/lib/logstash/outputs/sumologic/payload_builder.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
-require "logstash/json"
-require "logstash/event"
-
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class PayloadBuilder
     
+    require "logstash/json"
+    require "logstash/event"
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     TIMESTAMP_FIELD = "@timestamp"

data/lib/logstash/outputs/sumologic/piler.rb

@@ -1,11 +1,11 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Piler
 
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :is_pile
@@ -17,14 +17,14 @@ module LogStash; module Outputs; class SumoLogic;
       @queue = queue
       @stats = stats
       @stopping = Concurrent::AtomicBoolean.new(false)
+      @payload_builder = PayloadBuilder.new(@stats, config)
+      @header_builder = HeaderBuilder.new(config)
       @is_pile = (@interval > 0 && @pile_max > 0)
-
       if (@is_pile)
-        @pile = Array.new
-        @pile_size = 0
+        @pile = Hash.new("")
         @semaphore = Mutex.new
       end
-
+  
     end # def initialize
 
     def start()
@@ -46,45 +46,43 @@ module LogStash; module Outputs; class SumoLogic;
     def stop()
       @stopping.make_true()
       if (@is_pile)
-        log_info("shutting down piler...")
-        @piler_t.join
+        log_info("shutting down piler in #{@interval * 2} secs ...")
+        @piler_t.join(@interval * 2)
         log_info("piler is fully shutted down")
       end
     end # def stop
 
-    def input(entry)
+    def input(event)
       if (@stopping.true?)
-        log_warn("piler is shutting down, message dropped", 
-          "message" => entry)
-      elsif (@is_pile)
-        @semaphore.synchronize {
-          if @pile_size + entry.bytesize > @pile_max
-            @queue.enq(@pile.join($/))
-            @pile.clear
-            @pile_size = 0
-            @stats.record_clear_pile()
-          end
-          @pile << entry
-          @pile_size += entry.bytesize
-          @stats.record_input(entry)
-        }
+        log_warn("piler is shutting down, event is dropped", 
+          "event" => event)
       else
-        @queue.enq(entry)
-      end # if
+        headers = @header_builder.build(event)
+        payload = @payload_builder.build(event)
+        if (@is_pile)
+          @semaphore.synchronize {
+            content = @pile[headers]
+            size = content.bytesize
+            if size + payload.bytesize > @pile_max
+              @queue.enq(Batch.new(headers, content))
+              @pile[headers] = ""
+            end
+            @pile[headers] = @pile[headers].blank? ? payload : "#{@pile[headers]}\n#{payload}"
+          }
+        else
+          @queue.enq(Batch.new(headers, payload))
+        end # if
+      end
     end # def input
 
     private
     def enq_and_clear()
-      if (@pile.size > 0)
-        @semaphore.synchronize {
-          if (@pile.size > 0)
-            @queue.enq(@pile.join($/))
-            @pile.clear
-            @pile_size = 0
-            @stats.record_clear_pile()
-          end
-        }
-      end
+      @semaphore.synchronize {
+        @pile.each do |headers, content|
+          @queue.enq(Batch.new(headers, content))
+        end
+        @pile.clear()
+      }
     end # def enq_and_clear
 
   end

data/lib/logstash/outputs/sumologic/sender.rb

@@ -1,19 +1,19 @@
 # encoding: utf-8
-require "net/https"
-require "socket"
-require "thread"
-require "uri"
-require "logstash/outputs/sumologic/common"
-require "logstash/outputs/sumologic/compressor"
-require "logstash/outputs/sumologic/header_builder"
-require "logstash/outputs/sumologic/statistics"
-require "logstash/outputs/sumologic/message_queue"
 
 module LogStash; module Outputs; class SumoLogic;
   class Sender
 
+    require "net/https"
+    require "socket"
+    require "thread"
+    require "uri"
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/compressor"
+    require "logstash/outputs/sumologic/header_builder"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
     include LogStash::Outputs::SumoLogic::Common
-    STOP_TAG = "PLUGIN STOPPED"
+
 
     def initialize(client, queue, stats, config)
       @client = client
@@ -28,9 +28,6 @@ module LogStash; module Outputs; class SumoLogic;
       @tokens = SizedQueue.new(@sender_max)
       @sender_max.times { |t| @tokens << t }
 
-      @header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
-      @headers = @header_builder.build()
-      @stats_headers = @header_builder.build_stats()
       @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
 
     end # def initialize
@@ -42,11 +39,11 @@ module LogStash; module Outputs; class SumoLogic;
       @stopping.make_false()
       @sender_t = Thread.new {
         while @stopping.false?
-          content = @queue.deq()
-          send_request(content)
+          batch = @queue.deq()
+          send_request(batch)
         end # while
-        @queue.drain().map { |content| 
-          send_request(content)
+        @queue.drain().map { |batch| 
+          send_request(batch)
         }
         log_info("waiting while senders finishing...")
         while @tokens.size < @sender_max
@@ -58,7 +55,7 @@ module LogStash; module Outputs; class SumoLogic;
     def stop()
       log_info("shutting down sender...")
       @stopping.make_true()
-      @queue.enq(STOP_TAG)
+      @queue.enq(Batch.new(Hash.new, STOP_TAG))
       @sender_t.join
       log_info("sender is fully shutted down")
     end # def stop
@@ -93,7 +90,9 @@ module LogStash; module Outputs; class SumoLogic;
     
     private
 
-    def send_request(content)
+    def send_request(batch)
+      content = batch.payload
+      headers = batch.headers
       if content == STOP_TAG
         log_info("STOP_TAG is received.")
         return
@@ -103,10 +102,10 @@ module LogStash; module Outputs; class SumoLogic;
 
       if @stats_enabled && content.start_with?(STATS_TAG)
         body = @compressor.compress(content[STATS_TAG.length..-1])
-        headers = @stats_headers
+        headers[CATEGORY_HEADER] = "#{headers[CATEGORY_HEADER]}.stats"
+        headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
       else
         body = @compressor.compress(content)
-        headers = @headers
       end
 
       log_dbg("sending request",
@@ -129,7 +128,7 @@ module LogStash; module Outputs; class SumoLogic;
             :headers => headers,
             :contet => content[0..20])
           if response.code == 429 || response.code == 503 || response.code == 504
-            requeue_message(content)
+            requeue_message(batch)
           end
         else
           log_dbg("request accepted",
@@ -145,22 +144,27 @@ module LogStash; module Outputs; class SumoLogic;
           :message => exception.message,
           :class => exception.class.name,
           :backtrace => exception.backtrace)
-        requeue_message(content)
+        requeue_message(batch)
       end      
 
       @stats.record_request(content.bytesize, body.bytesize)
       request.call
     end # def send_request
 
-    def requeue_message(content)
-      if @stopping.false? && @sleep_before_requeue >= 0
+    def requeue_message(batch)
+      content = batch.payload
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        log_warn("do not requeue stats payload",
+          :content => content)
+      elsif @stopping.false? && @sleep_before_requeue >= 0
         log_info("requeue message",
           :after => @sleep_before_requeue,
           :queue_size => @queue.size,
           :content_size => content.size,
-          :content => content[0..20])
+          :content => content[0..20],
+          :headers => batch.headers)
         Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
-        @queue.enq(content)
+        @queue.enq(batch)
       end
     end # def reque_message
 

data/lib/logstash/outputs/sumologic/statistics.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class Statistics
 
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :initialize_time
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
     attr_reader :total_input_bytes
     attr_reader :total_metrics_datapoints
     attr_reader :total_log_lines
-    attr_reader :current_pile_items
-    attr_reader :current_pile_bytes
     attr_reader :total_enque_times
     attr_reader :total_enque_bytes
     attr_reader :total_deque_times
     attr_reader :total_deque_bytes
-    attr_reader :current_queue_items
-    attr_reader :current_queue_bytes
     attr_reader :total_output_requests
     attr_reader :total_output_bytes
     attr_reader :total_output_bytes_compressed 
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
       @total_input_bytes = Concurrent::AtomicFixnum.new
       @total_metrics_datapoints = Concurrent::AtomicFixnum.new
       @total_log_lines = Concurrent::AtomicFixnum.new
-      @current_pile_items = Concurrent::AtomicFixnum.new
-      @current_pile_bytes = Concurrent::AtomicFixnum.new
       @total_enque_times = Concurrent::AtomicFixnum.new
       @total_enque_bytes = Concurrent::AtomicFixnum.new
       @total_deque_times = Concurrent::AtomicFixnum.new
       @total_deque_bytes = Concurrent::AtomicFixnum.new
-      @current_queue_items = Concurrent::AtomicFixnum.new
-      @current_queue_bytes = Concurrent::AtomicFixnum.new
       @total_output_requests = Concurrent::AtomicFixnum.new
       @total_output_bytes = Concurrent::AtomicFixnum.new
       @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
       @total_response.get(key) ? @total_response.get(key).value : 0
     end
 
-    def record_multi_input(events, bytesize)
-      @total_input_events.update { |v| v + events }
-      @total_input_bytes.update { |v| v + bytesize }
-    end # def record_multi_input
-
-    def record_input(entry)
+    def record_input(size)
       @total_input_events.increment()
-      @total_input_bytes.update { |v| v + entry.bytesize }
-      @current_pile_items.increment()
-      @current_pile_bytes.update { |v| v + entry.bytesize }
+      @total_input_bytes.update { |v| v + size }
     end # def record_input
 
     def record_log_process()
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
       @total_metrics_datapoints.update { |v| v + dps }
     end # def record_metrics_process
 
-    def record_clear_pile()
-      @current_pile_items.value= 0
-      @current_pile_bytes.value= 0
-    end # def record_pile_clear
-
-    def record_enque(payload)
+    def record_enque(size)
       @total_enque_times.increment()
-      @total_enque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.increment()
-      @current_queue_bytes.update { |v| v + payload.bytesize }
+      @total_enque_bytes.update { |v| v + size }
     end # def record_enque
 
-    def record_deque(payload)
+    def record_deque(size)
       @total_deque_times.increment()
-      @total_deque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.decrement()
-      @current_queue_bytes.update { |v| v - payload.bytesize }
+      @total_deque_bytes.update { |v| v + size }
     end # def record_deque
 
     def record_request(size, size_compressed)

data/logstash-output-sumologic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sumologic'
-  s.version       = '1.2.2'
+  s.version       = '1.3.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Deliever the log to Sumo Logic cloud service.'
   s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-mixin-http_client'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '~> 6.0'
   
+  s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
 describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   result = {}
+  event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
 
   before :each do
-    result = builder.build()
+    result = builder.build(event)
   end
 
   context "should build headers by default" do
+
     let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
 
     specify {
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_category with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category bar")
+    }
+
+  end # context
+
   context "should override source_name" do
     
     let(:builder) {
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_name with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name bar")
+    }
+
+  end # context
+
   context "should override source_host" do
     
     let(:builder) {
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_host with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host bar")
+    }
+
+  end # context
+
   context "should hornor extra_headers" do
     
     let(:builder) {

data/spec/outputs/sumologic/message_queue_spec.rb

@@ -1,46 +1,48 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sumologic"
+include LogStash::Outputs
+
+describe SumoLogic::MessageQueue do
 
-describe LogStash::Outputs::SumoLogic::MessageQueue do
-  
   context "working in pile mode if interval > 0 && pile_max > 0" do
 
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
+    let(:stats) { SumoLogic::Statistics.new }
 
     it "enq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
         expect(queue.size()).to eq(i + 1)
         expect(stats.total_enque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(100)
     end
 
     it "deq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       10.times { |i|
         expect(queue.size()).to eq(10 - i)
         result = queue.deq()
-        expect(result).to eq("test - #{i}")
+        expect(result.payload).to eq("test  -  #{i}")
         expect(stats.total_deque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(0)
     end
 
     it "drain() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       result = queue.drain()
       expect(queue.size()).to eq(0)
       expect(stats.total_deque_times.value).to eq(10)
       expect(result.size).to eq(10)
+      expect(queue.bytesize()).to eq(0)
       10.times { |i|
-        expect(result[i]).to eq("test - #{i}")
+        expect(result[i].payload).to eq("test  -  #{i}")
       }
     end
 

data/spec/outputs/sumologic/piler_spec.rb

@@ -1,9 +1,13 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sumologic"
+include LogStash::Outputs
 
-describe LogStash::Outputs::SumoLogic::Piler do
+describe SumoLogic::Piler do
   
+  event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
+  event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
+
   before :each do
     piler.start()
   end
@@ -15,9 +19,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in pile mode if interval > 0 && pile_max > 0" do
     let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be true
     }
@@ -25,9 +29,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in non-pile mode if interval <= 0" do
     let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be false
     }
@@ -35,152 +39,113 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in non-pile mode if pile_max <= 0" do
     let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be false
     }
   end # context
 
   context "in non-pile mode" do
-
-    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "enque immediately after input" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("This is a log line")
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 18
+      expect(stats.total_enque_times.value).to be 0
+      expect(queue.size).to be 0
+      piler.input(event)
+      expect(stats.total_enque_times.value).to be 1
+      expect(stats.total_enque_bytes.value).to be 18
+      expect(queue.size).to be 1
+      expect(queue.bytesize).to be 18
     end
 
     it "deque correctly" do
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.total_enque_times.value).to be 0
-      piler.input("This is a log line")
-      expect(stats.total_enque_times.value).to be 1
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 18
-      expect(stats.total_deque_times.value).to be 0
-      expect(queue.deq()).to eq "This is a log line"
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.current_queue_bytes.value).to be 0
+      piler.input(event)
+      expect(queue.deq().payload).to eq "This is a log line"
+      expect(queue.size).to be 0
+      expect(queue.bytesize).to be 0
       expect(stats.total_deque_times.value).to be 1
+      expect(stats.total_deque_bytes.value).to be 18
     end
 
   end # context
 
   context "in pile mode" do
 
-    let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
-
-    it "wait in pile before size reach pile_max" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 1
-      expect(stats.current_pile_bytes.value).to be 10
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 2
-      expect(stats.current_pile_bytes.value).to be 20
-      expect(stats.current_queue_items.value).to be 0
-    end
+    let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "enqueue content from pile when reach pile_max" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 1
-      expect(stats.current_pile_bytes.value).to be 10
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 21
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 1
     end
 
     it "enqueue content from pile when reach interval" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      piler.input("1234567890")
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      sleep(8)
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 21
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      sleep(10)
+      expect(queue.size).to be 1
     end
 
   end # context
 
   context "pile to message queue" do
 
-    let(:config) { {"queue_max" => 5, "interval" => 500, "pile_max" => 5} }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
-
-    it "enqueue payloads from pile before reach queue_max" do
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.current_queue_bytes.value).to be 0
-      piler.input("2234567890")
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 10
-      piler.input("3234567890")
-      expect(stats.current_queue_items.value).to be 2
-      expect(stats.current_queue_bytes.value).to be 20
-    end
+    let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "block input thread if queue is full" do
       input_t = Thread.new {
-        for i in 0..9 do
-          piler.input("#{i}234567890")
+        for i in 0..10 do
+          piler.input(event_10)
         end
       }
       sleep(3)
-      expect(stats.current_queue_items.value).to be 5
-      expect(stats.current_queue_bytes.value).to be 50
-      queue.drain()
+      expect(queue.size).to be 5
+      expect(queue.bytesize).to be 50
       piler.stop()
-      input_t.join
-      expect(stats.total_deque_times.value).to be 5
-      expect(stats.total_deque_bytes.value).to be 50
+      queue.drain()
+      input_t.kill()
     end
 
     it "resume input thread if queue is drained" do
       input_t = Thread.new {
-        for i in 0..9 do
-          piler.input("#{i}234567890")
+        for i in 0..10 do
+          piler.input(event_10)
         end
       }
-      sleep(3)
+      sleep(5)
       expect(stats.total_deque_times.value).to be 0
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 5
       queue.deq()
-      sleep(1)
+      sleep(3)
       expect(stats.total_deque_times.value).to be 1
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 6
       queue.deq()
-      sleep(1)
+      sleep(3)
       expect(stats.total_deque_times.value).to be 2
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 7
-      queue.drain()
       piler.stop()
-      input_t.join
+      queue.drain()
+      input_t.kill()
     end
 
   end # context

data/spec/outputs/sumologic_spec.rb

@@ -13,6 +13,16 @@ describe LogStash::Outputs::SumoLogic, :unless => (ENV["sumo_url"].to_s.empty?)
     plugin.close()
   end
 
+  context "default configuration" do
+    let(:plugin) {
+      LogStash::Outputs::SumoLogic.new("url" => ENV["sumo_url"])
+    }
+    
+    it "cookies is by default disabled" do
+      expect(plugin.cookies).to be false
+    end
+  end
+  
   context "no pile" do
     context "single sender" do
       context "send log in json" do

metadata

@@ -1,17 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sumologic
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Sumo Logic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2020-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: manticore
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -20,8 +19,9 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-  type: :runtime
+  name: manticore
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -31,7 +31,6 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -40,8 +39,9 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  type: :runtime
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,42 +51,42 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '6.0'
+  name: logstash-mixin-http_client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: logstash-mixin-http_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -106,6 +106,7 @@ files:
 - LICENSE
 - README.md
 - lib/logstash/outputs/sumologic.rb
+- lib/logstash/outputs/sumologic/batch.rb
 - lib/logstash/outputs/sumologic/common.rb
 - lib/logstash/outputs/sumologic/compressor.rb
 - lib/logstash/outputs/sumologic/header_builder.rb
@@ -130,7 +131,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -145,8 +146,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.0
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
 specification_version: 4
 summary: Deliever the log to Sumo Logic cloud service.
 test_files: