logstash-output-sumologic 1.1.9 → 1.3.1

data/lib/logstash/outputs/sumologic/statistics.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
-require "logstash/outputs/sumologic/common"
 
 module LogStash; module Outputs; class SumoLogic;
   class Statistics
 
+    require "logstash/outputs/sumologic/common"
     include LogStash::Outputs::SumoLogic::Common
 
     attr_reader :initialize_time
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
     attr_reader :total_input_bytes
     attr_reader :total_metrics_datapoints
     attr_reader :total_log_lines
-    attr_reader :current_pile_items
-    attr_reader :current_pile_bytes
     attr_reader :total_enque_times
     attr_reader :total_enque_bytes
     attr_reader :total_deque_times
     attr_reader :total_deque_bytes
-    attr_reader :current_queue_items
-    attr_reader :current_queue_bytes
     attr_reader :total_output_requests
     attr_reader :total_output_bytes
     attr_reader :total_output_bytes_compressed 
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
       @total_input_bytes = Concurrent::AtomicFixnum.new
       @total_metrics_datapoints = Concurrent::AtomicFixnum.new
       @total_log_lines = Concurrent::AtomicFixnum.new
-      @current_pile_items = Concurrent::AtomicFixnum.new
-      @current_pile_bytes = Concurrent::AtomicFixnum.new
       @total_enque_times = Concurrent::AtomicFixnum.new
       @total_enque_bytes = Concurrent::AtomicFixnum.new
       @total_deque_times = Concurrent::AtomicFixnum.new
       @total_deque_bytes = Concurrent::AtomicFixnum.new
-      @current_queue_items = Concurrent::AtomicFixnum.new
-      @current_queue_bytes = Concurrent::AtomicFixnum.new
       @total_output_requests = Concurrent::AtomicFixnum.new
       @total_output_bytes = Concurrent::AtomicFixnum.new
       @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
       @total_response.get(key) ? @total_response.get(key).value : 0
     end
 
-    def record_multi_input(events, bytesize)
-      @total_input_events.update { |v| v + events }
-      @total_input_bytes.update { |v| v + bytesize }
-    end # def record_multi_input
-
-    def record_input(entry)
+    def record_input(size)
       @total_input_events.increment()
-      @total_input_bytes.update { |v| v + entry.bytesize }
-      @current_pile_items.increment()
-      @current_pile_bytes.update { |v| v + entry.bytesize }
+      @total_input_bytes.update { |v| v + size }
     end # def record_input
 
     def record_log_process()
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
       @total_metrics_datapoints.update { |v| v + dps }
     end # def record_metrics_process
 
-    def record_clear_pile()
-      @current_pile_items.value= 0
-      @current_pile_bytes.value= 0
-    end # def record_pile_clear
-
-    def record_enque(payload)
+    def record_enque(size)
       @total_enque_times.increment()
-      @total_enque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.increment()
-      @current_queue_bytes.update { |v| v + payload.bytesize }
+      @total_enque_bytes.update { |v| v + size }
     end # def record_enque
 
-    def record_deque(payload)
+    def record_deque(size)
       @total_deque_times.increment()
-      @total_deque_bytes.update { |v| v + payload.bytesize }
-      @current_queue_items.decrement()
-      @current_queue_bytes.update { |v| v - payload.bytesize }
+      @total_deque_bytes.update { |v| v + size }
     end # def record_deque
 
     def record_request(size, size_compressed)

data/logstash-output-sumologic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sumologic'
-  s.version       = '1.1.9'
+  s.version       = '1.3.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Deliever the log to Sumo Logic cloud service.'
   s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-mixin-http_client'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '>= 6', '< 8'
   
+  s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
 describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   result = {}
+  event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
 
   before :each do
-    result = builder.build()
+    result = builder.build(event)
   end
 
   context "should build headers by default" do
+
     let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
 
     specify {
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_category with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category bar")
+    }
+
+  end # context
+
   context "should override source_name" do
     
     let(:builder) {
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_name with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name bar")
+    }
+
+  end # context
+
   context "should override source_host" do
     
     let(:builder) {
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
 
   end # context
 
+  context "should override source_host with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host bar")
+    }
+
+  end # context
+
   context "should hornor extra_headers" do
     
     let(:builder) {

data/spec/outputs/sumologic/message_queue_spec.rb

@@ -1,46 +1,48 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sumologic"
+include LogStash::Outputs
+
+describe SumoLogic::MessageQueue do
 
-describe LogStash::Outputs::SumoLogic::MessageQueue do
-  
   context "working in pile mode if interval > 0 && pile_max > 0" do
 
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
+    let(:stats) { SumoLogic::Statistics.new }
 
     it "enq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
         expect(queue.size()).to eq(i + 1)
         expect(stats.total_enque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(100)
     end
 
     it "deq() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       10.times { |i|
         expect(queue.size()).to eq(10 - i)
         result = queue.deq()
-        expect(result).to eq("test - #{i}")
+        expect(result.payload).to eq("test  -  #{i}")
         expect(stats.total_deque_times.value).to eq(i + 1)
       }
+      expect(queue.bytesize()).to eq(0)
     end
 
     it "drain() correctly" do
-      queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
       10.times { |i|
-        queue.enq("test - #{i}")
+        queue.enq(SumoLogic::Batch.new(Hash.new, "test  -  #{i}"))
       }
       result = queue.drain()
       expect(queue.size()).to eq(0)
       expect(stats.total_deque_times.value).to eq(10)
       expect(result.size).to eq(10)
+      expect(queue.bytesize()).to eq(0)
       10.times { |i|
-        expect(result[i]).to eq("test - #{i}")
+        expect(result[i].payload).to eq("test  -  #{i}")
       }
     end
 

data/spec/outputs/sumologic/payload_builder_spec.rb

@@ -69,14 +69,13 @@ describe LogStash::Outputs::SumoLogic::PayloadBuilder do
         "format" => "%{host} %{@json}",
         "json_mapping" => {
           "foo" => "%{foo}",
-          "bar" => "%{bar}",
-          "%{foo}" => "%{bar}"
+          "bar" => "%{bar}"
         })
     }
     let(:event) { LogStash::Event.new("host" => "myHost", "foo" => "fancy", "bar" => 24) }
 
     specify {
-      expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\",\"fancy\":\"24\"}")
+      expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\"}")
     }
 
   end # context

data/spec/outputs/sumologic/piler_spec.rb

@@ -1,9 +1,13 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sumologic"
+include LogStash::Outputs
 
-describe LogStash::Outputs::SumoLogic::Piler do
+describe SumoLogic::Piler do
   
+  event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
+  event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
+
   before :each do
     piler.start()
   end
@@ -15,9 +19,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in pile mode if interval > 0 && pile_max > 0" do
     let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be true
     }
@@ -25,9 +29,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in non-pile mode if interval <= 0" do
     let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be false
     }
@@ -35,152 +39,113 @@ describe LogStash::Outputs::SumoLogic::Piler do
 
   context "working in non-pile mode if pile_max <= 0" do
     let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
     specify {
       expect(piler.is_pile).to be false
     }
   end # context
 
   context "in non-pile mode" do
-
-    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
+    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "enque immediately after input" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("This is a log line")
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 18
+      expect(stats.total_enque_times.value).to be 0
+      expect(queue.size).to be 0
+      piler.input(event)
+      expect(stats.total_enque_times.value).to be 1
+      expect(stats.total_enque_bytes.value).to be 18
+      expect(queue.size).to be 1
+      expect(queue.bytesize).to be 18
     end
 
     it "deque correctly" do
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.total_enque_times.value).to be 0
-      piler.input("This is a log line")
-      expect(stats.total_enque_times.value).to be 1
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 18
-      expect(stats.total_deque_times.value).to be 0
-      expect(queue.deq()).to eq "This is a log line"
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.current_queue_bytes.value).to be 0
+      piler.input(event)
+      expect(queue.deq().payload).to eq "This is a log line"
+      expect(queue.size).to be 0
+      expect(queue.bytesize).to be 0
       expect(stats.total_deque_times.value).to be 1
+      expect(stats.total_deque_bytes.value).to be 18
     end
 
   end # context
 
   context "in pile mode" do
 
-    let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25 } }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
-
-    it "wait in pile before size reach pile_max" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 1
-      expect(stats.current_pile_bytes.value).to be 10
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 2
-      expect(stats.current_pile_bytes.value).to be 20
-      expect(stats.current_queue_items.value).to be 0
-    end
+    let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "enqueue content from pile when reach pile_max" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_pile_items.value).to be 1
-      expect(stats.current_pile_bytes.value).to be 10
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 21
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 1
     end
 
     it "enqueue content from pile when reach interval" do
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      piler.input("1234567890")
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      sleep(8)
-      expect(stats.current_pile_items.value).to be 0
-      expect(stats.current_pile_bytes.value).to be 0
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 21
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      sleep(10)
+      expect(queue.size).to be 1
     end
 
   end # context
 
   context "pile to message queue" do
 
-    let(:config) { {"queue_max" => 5, "interval" => 500, "pile_max" => 5} }
-    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
-    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
-    let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
-
-    it "enqueue payloads from pile before reach queue_max" do
-      expect(stats.current_queue_items.value).to be 0
-      piler.input("1234567890")
-      expect(stats.current_queue_items.value).to be 0
-      expect(stats.current_queue_bytes.value).to be 0
-      piler.input("2234567890")
-      expect(stats.current_queue_items.value).to be 1
-      expect(stats.current_queue_bytes.value).to be 10
-      piler.input("3234567890")
-      expect(stats.current_queue_items.value).to be 2
-      expect(stats.current_queue_bytes.value).to be 20
-    end
+    let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
 
     it "block input thread if queue is full" do
       input_t = Thread.new {
-        for i in 0..9 do
-          piler.input("#{i}234567890")
+        for i in 0..10 do
+          piler.input(event_10)
         end
       }
       sleep(3)
-      expect(stats.current_queue_items.value).to be 5
-      expect(stats.current_queue_bytes.value).to be 50
-      queue.drain()
+      expect(queue.size).to be 5
+      expect(queue.bytesize).to be 50
       piler.stop()
-      input_t.join
-      expect(stats.total_deque_times.value).to be 5
-      expect(stats.total_deque_bytes.value).to be 50
+      queue.drain()
+      input_t.kill()
     end
 
     it "resume input thread if queue is drained" do
       input_t = Thread.new {
-        for i in 0..9 do
-          piler.input("#{i}234567890")
+        for i in 0..10 do
+          piler.input(event_10)
         end
       }
-      sleep(3)
+      sleep(5)
       expect(stats.total_deque_times.value).to be 0
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 5
       queue.deq()
-      sleep(1)
+      sleep(3)
       expect(stats.total_deque_times.value).to be 1
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 6
       queue.deq()
-      sleep(1)
+      sleep(3)
       expect(stats.total_deque_times.value).to be 2
+      expect(queue.size).to be 5
       expect(stats.total_enque_times.value).to be 7
-      queue.drain()
       piler.stop()
-      input_t.join
+      queue.drain()
+      input_t.kill()
     end
 
   end # context