hshek-logstash-output-sumologic 0.0.2

data/spec/outputs/sumologic/piler_spec.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+include LogStash::Outputs
+
+describe SumoLogic::Piler do
+  
+  event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
+  event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
+
+  before :each do
+    piler.start()
+  end
+
+  after :each do
+    queue.drain()
+    piler.stop()
+  end
+
+  context "working in pile mode if interval > 0 && pile_max > 0" do
+    let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+    specify {
+      expect(piler.is_pile).to be true
+    }
+  end # context
+
+  context "working in non-pile mode if interval <= 0" do
+    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+    specify {
+      expect(piler.is_pile).to be false
+    }
+  end # context
+
+  context "working in non-pile mode if pile_max <= 0" do
+    let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+    specify {
+      expect(piler.is_pile).to be false
+    }
+  end # context
+
+  context "in non-pile mode" do
+    let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+
+    it "enque immediately after input" do
+      expect(stats.total_enque_times.value).to be 0
+      expect(queue.size).to be 0
+      piler.input(event)
+      expect(stats.total_enque_times.value).to be 1
+      expect(stats.total_enque_bytes.value).to be 18
+      expect(queue.size).to be 1
+      expect(queue.bytesize).to be 18
+    end
+
+    it "deque correctly" do
+      piler.input(event)
+      expect(queue.deq().payload).to eq "This is a log line"
+      expect(queue.size).to be 0
+      expect(queue.bytesize).to be 0
+      expect(stats.total_deque_times.value).to be 1
+      expect(stats.total_deque_bytes.value).to be 18
+    end
+
+  end # context
+
+  context "in pile mode" do
+
+    let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+
+    it "enqueue content from pile when reach pile_max" do
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 1
+    end
+
+    it "enqueue content from pile when reach interval" do
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      expect(queue.size).to be 0
+      piler.input(event_10)
+      sleep(10)
+      expect(queue.size).to be 1
+    end
+
+  end # context
+
+  context "pile to message queue" do
+
+    let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
+    let(:stats) { SumoLogic::Statistics.new }
+    let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
+    let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
+
+    it "block input thread if queue is full" do
+      input_t = Thread.new {
+        for i in 0..10 do
+          piler.input(event_10)
+        end
+      }
+      sleep(3)
+      expect(queue.size).to be 5
+      expect(queue.bytesize).to be 50
+      piler.stop()
+      queue.drain()
+      input_t.kill()
+    end
+
+    it "resume input thread if queue is drained" do
+      input_t = Thread.new {
+        for i in 0..10 do
+          piler.input(event_10)
+        end
+      }
+      sleep(5)
+      expect(stats.total_deque_times.value).to be 0
+      expect(queue.size).to be 5
+      expect(stats.total_enque_times.value).to be 5
+      queue.deq()
+      sleep(3)
+      expect(stats.total_deque_times.value).to be 1
+      expect(queue.size).to be 5
+      expect(stats.total_enque_times.value).to be 6
+      queue.deq()
+      sleep(3)
+      expect(stats.total_deque_times.value).to be 2
+      expect(queue.size).to be 5
+      expect(stats.total_enque_times.value).to be 7
+      piler.stop()
+      queue.drain()
+      input_t.kill()
+    end
+
+  end # context
+
+end # describe
+

data/spec/outputs/sumologic/sender_spec.rb

@@ -0,0 +1,188 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "rspec/eventually"
+require "logstash/outputs/sumologic"
+
+require_relative "../../test_server.rb"
+
+describe LogStash::Outputs::SumoLogic::Sender do
+
+  before :all do
+    @@server = TestServer.new()
+    @@server.start()
+  end
+
+  before :each do
+    @@server.response = TestServer::RESPONSE_200
+    @@server.drain()
+  end
+
+  after :all do
+    @@server.stop()
+  end
+
+  context "connect()" do
+    let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
+    let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
+    let(:sender) { LogStash::Outputs::SumoLogic::Sender.new(false, queue, stats, "url" => "http://localhost:#{TestServer::PORT}") }
+
+    it "should return true if sever response 200" do
+      expect(sender.connect()).to be true
+      result = @@server.drain()
+      expect(result.size).to eq(1)
+    end
+
+    it "should return false if sever response 429" do
+      @@server.response = TestServer::RESPONSE_429
+      expect(sender.connect()).to be false
+      result = @@server.drain()
+      expect(result.size).to eq(1)
+    end
+
+    it "should return false if sever cannot reach" do
+      sender = LogStash::Outputs::SumoLogic::Sender.new(false, queue, stats, "url" => "http://localhost:#{TestServer::PORT + 1}")
+      expect(sender.connect()).to be false
+      result = @@server.drain()
+      expect(result.size).to eq(0)
+    end
+
+  end # context
+
+  context "single sender" do
+    let(:plugin) { LogStash::Outputs::SumoLogic.new("url" => "http://localhost:#{TestServer::PORT}", "sender_max" => 1, "sleep_before_requeue" => 1, "request_timeout" => 3) }
+    let(:event) { LogStash::Event.new("host" => "myHost", "message" => "Hello world") }
+
+    it "should send message correctly" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(2)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(2)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(1)
+    end
+
+    it "should re-enque the message if got 429" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      @@server.response = TestServer::RESPONSE_429
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(be > 1).within(10).pause_for(1)
+      expect { plugin.stats.total_response("429") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+      @@server.response = TestServer::RESPONSE_200
+      expect { plugin.stats.total_response("200") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+    end
+
+    it "should re-enque the message if network failed" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      @@server.stop()
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(be > 1).within(10).pause_for(1)
+      expect { plugin.stats.total_response("failure") }.to eventually(be > 1).within(25).pause_for(1)
+      result = @@server.drain()
+      @@server.start()
+      @@server.response = TestServer::RESPONSE_200
+      expect { plugin.stats.total_response("200") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+    end
+
+  end # context
+
+  context "multiple senders" do
+    let(:plugin) { LogStash::Outputs::SumoLogic.new("url" => "http://localhost:#{TestServer::PORT}", "sender_max" => 10, "sleep_before_requeue" => 1, "request_timeout" => 3) }
+    let(:event) { LogStash::Event.new("host" => "myHost", "message" => "Hello world") }
+
+    it "should send message correctly" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(2)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(2)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(1)
+    end
+
+    it "should re-enque the message if got 429" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      @@server.response = TestServer::RESPONSE_429
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(be > 1).within(10).pause_for(1)
+      expect { plugin.stats.total_response("429") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+      @@server.response = TestServer::RESPONSE_200
+      expect { plugin.stats.total_response("200") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+    end
+    
+    it "should re-enque the message if network failed" do
+      plugin.register
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(eq(1)).pause_for(1)
+      expect { plugin.stats.total_response("200") }.to eventually(eq(1)).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to eq(2)
+      @@server.stop()
+      plugin.receive(event)
+      expect { plugin.stats.total_output_requests.value }.to eventually(be > 1).within(10).pause_for(1)
+      expect { plugin.stats.total_response("failure") }.to eventually(be > 1).within(25).pause_for(1)
+      result = @@server.drain()
+      @@server.start()
+      @@server.response = TestServer::RESPONSE_200
+      expect { plugin.stats.total_response("200") }.to eventually(be > 1).within(10).pause_for(1)
+      result = @@server.drain()
+      expect(result.size).to be > 0
+    end
+
+    it "should reuse tokens" do
+      plugin.register
+      30.times { plugin.receive(event) }
+      expect { plugin.stats.total_response("200") }.to eventually(eq 30).within(100).pause_for(1)
+    end
+
+  end # context
+
+  context "close()" do
+
+    let(:plugin) { LogStash::Outputs::SumoLogic.new("url" => "http://localhost:#{TestServer::PORT}", "sender_max" => 10) }
+    let(:event) { LogStash::Event.new("host" => "myHost", "message" => "Hello world") }
+
+    it "should drain out messages" do
+      plugin.register
+      30.times { plugin.receive(event) }
+      plugin.close
+      expect(plugin.stats.total_response("200")).to eq 30
+    end
+
+  end # context
+
+end # describe
+

data/spec/outputs/sumologic_spec.rb

@@ -0,0 +1,240 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "rspec/eventually"
+require "logstash/outputs/sumologic"
+
+describe LogStash::Outputs::SumoLogic, :unless => (ENV["sumo_url"].to_s.empty?) do
+
+  before :each do
+    plugin.register()
+  end
+
+  after :each do
+    plugin.close()
+  end
+
+  context "default configuration" do
+    let(:plugin) {
+      LogStash::Outputs::SumoLogic.new("url" => ENV["sumo_url"])
+    }
+    
+    it "cookies is by default disabled" do
+      expect(plugin.cookies).to be false
+    end
+  end
+  
+  context "no pile" do
+    context "single sender" do
+      context "send log in json" do
+        let(:plugin) { 
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 1,
+            "format" => "%{@json}")
+        }
+        specify {
+          event = LogStash::Event.new("host" => "myHost", "message" => "Hello world")
+          plugin.receive(event)
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+      context "send fields as metrics" do
+        let(:plugin) { 
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"],
+            "sender_max" => 1,
+            "fields_as_metrics" => true,
+            "intrinsic_tags" => {
+              "host"=>"%{host}"
+            },
+            "meta_tags" => {
+              "foo" => "%{foo}"
+            })
+        }
+
+        specify {
+          event = LogStash::Event.new(
+            "host" => "myHost",
+            "foo" => "fancy",
+            "cpu" => [0.24, 0.11, 0.75, 0.28],
+            "storageRW" => 51,
+            "bar" => "blahblah",
+            "blkio" => {
+              "write_ps" => 5,
+              "read_ps" => 2,
+              "total_ps" => 0
+            })
+          
+          plugin.receive(event)
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+    end
+    context "multiple senders" do
+      context "send log in json" do
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 5,
+            "format" => "%{@json}")
+        }
+        
+        specify {
+          5.times { |t| 
+            event = LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t}")
+            plugin.receive(event)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+
+      context "send multiple log in json" do
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 5,
+            "format" => "%{@json}"
+          )
+        }
+        
+        specify {
+          5.times { |t| 
+            events = 10.times.map { |r|
+              LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t} - #{r}")
+            }
+            plugin.multi_receive(events)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+    end
+  end
+  context "has pile" do
+    context "single sender" do
+      context "send log in json" do
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 1,
+            "interval" => 3,
+            "format" => "%{@json}")
+        }
+        
+        specify {
+          5.times { |t| 
+            event = LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t}")
+            plugin.receive(event)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+
+      context "send multiple log in json" do
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 1,
+            "interval" => 3,
+            "format" => "%{@json}"
+          )
+        }
+        
+        specify {
+          5.times { |t| 
+            events = 10.times.map { |r|
+              LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t} - #{r}")
+            }
+            plugin.multi_receive(events)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+    end
+    context "multi senders" do
+      context "send log in json" do
+        
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 5,
+            "interval" => 3,
+            "format" => "%{@json}")
+        }
+        
+        specify {
+          5.times { |t| 
+            event = LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t}")
+            plugin.receive(event)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+      context "send multiple log in json" do
+        let(:plugin) {
+          LogStash::Outputs::SumoLogic.new(
+            "url" => ENV["sumo_url"], 
+            "sender_max" => 5,
+            "interval" => 3,
+            "format" => "%{@json}"
+          )
+        }
+        
+        specify {
+          5.times { |t| 
+            events = 10.times.map { |r|
+              LogStash::Event.new("host" => "myHost", "message" => "Hello world - #{t} - #{r}")
+            }
+            plugin.multi_receive(events)
+          } 
+          expect { plugin.stats.total_output_requests.value }.to eventually(be > 0).within(10).pause_for(1)
+          expect { plugin.stats.total_response("200") }.to eventually(be > 0).within(10).pause_for(1)
+        }
+      end
+    end
+  end
+
+  @@map = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
+  def get_line(length)
+    length.times.map { @@map[rand(@@map.length)] }.join
+  end
+
+  context "throughput baseline" do
+    let(:plugin) {
+      LogStash::Outputs::SumoLogic.new(
+        "url" => ENV["sumo_url"], 
+        "source_category" => "logstash_ci_baseline",
+        "sender_max" => 100,
+        "interval" => 30,
+        "format" => "%{@timestamp} %{message}",
+        "compress" => true,
+        "compress_encoding" => "gzip",
+        "stats_enabled" => true,
+        "stats_interval" => 1
+      )
+    }
+
+    log_length = 5000 + rand(1000)
+    log_count = 50000 + rand(10000)
+  
+    specify {
+      log_count.times { |t| 
+        event = LogStash::Event.new("message" => "#{t} - #{get_line(log_length)}")
+        plugin.receive(event)
+      } 
+      expect { plugin.stats.total_log_lines.value }.to eventually(be >= log_count).within(60).pause_for(1)
+      bytes = plugin.stats.total_output_bytes.value
+      spend = (Time.now - plugin.stats.initialize_time) * 1000
+      rate = bytes / spend * 1000
+      puts "Sent #{plugin.stats.total_log_lines.value} log lines with #{bytes} bytes in #{'%.2f' % spend} ms, rate #{'%.2f' % (rate/1024/1024) } MB/s."
+      expect(rate).to be > 2_000_000
+    }
+  end
+
+end # describe