hshek-logstash-output-sumologic 0.0.2

data/lib/logstash/outputs/sumologic/piler.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Piler
+
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
+    include LogStash::Outputs::SumoLogic::Common
+
+    attr_reader :is_pile
+
+    def initialize(queue, stats, config)
+      
+      @interval = config["interval"] ||= 0
+      @pile_max = config["pile_max"] ||= 0
+      @queue = queue
+      @stats = stats
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @payload_builder = PayloadBuilder.new(@stats, config)
+      @header_builder = HeaderBuilder.new(config)
+      @is_pile = (@interval > 0 && @pile_max > 0)
+      if (@is_pile)
+        @pile = Hash.new("")
+        @semaphore = Mutex.new
+      end
+  
+    end # def initialize
+
+    def start()
+      @stopping.make_false()
+      if (@is_pile)
+        log_info("starting piler...", 
+          :max => @pile_max, 
+          :timeout => @interval)
+        @piler_t = Thread.new { 
+          while @stopping.false?
+            Stud.stoppable_sleep(@interval) { @stopping.true? }
+            log_dbg("timeout", :timeout => @interval)
+            enq_and_clear()
+          end # while
+        }
+      end # if
+    end # def start
+
+    def stop()
+      @stopping.make_true()
+      if (@is_pile)
+        log_info("shutting down piler in #{@interval * 2} secs ...")
+        @piler_t.join(@interval * 2)
+        log_info("piler is fully shutted down")
+      end
+    end # def stop
+
+    def input(event)
+      if (@stopping.true?)
+        log_warn("piler is shutting down, event is dropped", 
+          "event" => event)
+      else
+        headers = @header_builder.build(event)
+        payload = @payload_builder.build(event)
+        if (@is_pile)
+          @semaphore.synchronize {
+            content = @pile[headers]
+            size = content.bytesize
+            if size + payload.bytesize > @pile_max
+              @queue.enq(Batch.new(headers, content))
+              @pile[headers] = ""
+            end
+            @pile[headers] = @pile[headers].blank? ? payload : "#{@pile[headers]}\n#{payload}"
+          }
+        else
+          @queue.enq(Batch.new(headers, payload))
+        end # if
+      end
+    end # def input
+
+    private
+    def enq_and_clear()
+      @semaphore.synchronize {
+        @pile.each do |headers, content|
+          @queue.enq(Batch.new(headers, content))
+        end
+        @pile.clear()
+      }
+    end # def enq_and_clear
+
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/sender.rb

@@ -0,0 +1,172 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Sender
+
+    require "net/https"
+    require "socket"
+    require "thread"
+    require "uri"
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/compressor"
+    require "logstash/outputs/sumologic/header_builder"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
+    include LogStash::Outputs::SumoLogic::Common
+
+
+    def initialize(client, queue, stats, config)
+      @client = client
+      @queue = queue
+      @stats = stats
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @url = config["url"]
+      @sender_max = (config["sender_max"] ||= 1) < 1 ? 1 : config["sender_max"]
+      @sleep_before_requeue = config["sleep_before_requeue"] ||= 30
+      @stats_enabled = config["stats_enabled"] ||= false
+
+      @tokens = SizedQueue.new(@sender_max)
+      @sender_max.times { |t| @tokens << t }
+
+      @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
+
+    end # def initialize
+
+    def start()
+      log_info("starting sender...",
+        :max => @sender_max, 
+        :requeue => @sleep_before_requeue)
+      @stopping.make_false()
+      @sender_t = Thread.new {
+        while @stopping.false?
+          batch = @queue.deq()
+          send_request(batch)
+        end # while
+        @queue.drain().map { |batch| 
+          send_request(batch)
+        }
+        log_info("waiting while senders finishing...")
+        while @tokens.size < @sender_max
+          sleep 1
+        end # while
+      }
+    end # def start
+
+    def stop()
+      log_info("shutting down sender...")
+      @stopping.make_true()
+      @queue.enq(Batch.new(Hash.new, STOP_TAG))
+      @sender_t.join
+      log_info("sender is fully shutted down")
+    end # def stop
+    
+    def connect()
+      uri = URI.parse(@url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = @url.downcase().start_with?("https")
+      request = Net::HTTP::Get.new(uri.request_uri)
+      begin
+        res = http.request(request)
+        if res.code.to_i != 200
+          log_err("ping rejected",
+            :url => @url,
+            :code => res.code,
+            :body => res.body)
+          false
+        else
+          log_info("ping accepted",
+            :url => @url)
+          true
+        end
+      rescue Exception => exception
+        log_err("ping failed",
+          :url => @url,
+          :message => exception.message,
+          :class => exception.class.name,
+          :backtrace => exception.backtrace)
+        false
+      end
+    end # def connect
+    
+    private
+
+    def send_request(batch)
+      content = batch.payload
+      headers = batch.headers
+      if content == STOP_TAG
+        log_info("STOP_TAG is received.")
+        return
+      end
+      
+      token = @tokens.pop()
+
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        body = @compressor.compress(content[STATS_TAG.length..-1])
+        headers[CATEGORY_HEADER] = "#{headers[CATEGORY_HEADER]}.stats"
+        headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
+      else
+        body = @compressor.compress(content)
+      end
+
+      log_dbg("sending request",
+        :headers => headers,
+        :content_size => content.size,
+        :content => content[0..20],
+        :payload_size => body.size)
+      request = @client.send(:background).send(:post, @url, :body => body, :headers => headers)
+      
+      request.on_complete do
+        @tokens << token
+      end
+  
+      request.on_success do |response|
+        @stats.record_response_success(response.code)
+        if response.code < 200 || response.code > 299
+          log_err("request rejected",
+            :token => token,
+            :code => response.code,
+            :headers => headers,
+            :contet => content[0..20])
+          if response.code == 429 || response.code == 503 || response.code == 504
+            requeue_message(batch)
+          end
+        else
+          log_dbg("request accepted",
+            :token => token,
+            :code => response.code)
+        end
+      end
+  
+      request.on_failure do |exception|
+        @stats.record_response_failure()
+        log_err("error in network transmission",
+          :token => token,
+          :message => exception.message,
+          :class => exception.class.name,
+          :backtrace => exception.backtrace)
+        requeue_message(batch)
+      end      
+
+      @stats.record_request(content.bytesize, body.bytesize)
+      request.call
+    end # def send_request
+
+    def requeue_message(batch)
+      content = batch.payload
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        log_warn("do not requeue stats payload",
+          :content => content)
+      elsif @stopping.false? && @sleep_before_requeue >= 0
+        log_info("requeue message",
+          :after => @sleep_before_requeue,
+          :queue_size => @queue.size,
+          :content_size => content.size,
+          :content => content[0..20],
+          :headers => batch.headers)
+        Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
+        @queue.enq(batch)
+      end
+    end # def reque_message
+
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/statistics.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Statistics
+
+    require "logstash/outputs/sumologic/common"
+    include LogStash::Outputs::SumoLogic::Common
+
+    attr_reader :initialize_time
+    attr_reader :total_input_events
+    attr_reader :total_input_bytes
+    attr_reader :total_metrics_datapoints
+    attr_reader :total_log_lines
+    attr_reader :total_enque_times
+    attr_reader :total_enque_bytes
+    attr_reader :total_deque_times
+    attr_reader :total_deque_bytes
+    attr_reader :total_output_requests
+    attr_reader :total_output_bytes
+    attr_reader :total_output_bytes_compressed 
+    attr_reader :total_response
+    attr_reader :total_response_times
+    attr_reader :total_response_success
+
+    def initialize()
+      @initialize_time = Time.now()
+      @total_input_events = Concurrent::AtomicFixnum.new
+      @total_input_bytes = Concurrent::AtomicFixnum.new
+      @total_metrics_datapoints = Concurrent::AtomicFixnum.new
+      @total_log_lines = Concurrent::AtomicFixnum.new
+      @total_enque_times = Concurrent::AtomicFixnum.new
+      @total_enque_bytes = Concurrent::AtomicFixnum.new
+      @total_deque_times = Concurrent::AtomicFixnum.new
+      @total_deque_bytes = Concurrent::AtomicFixnum.new
+      @total_output_requests = Concurrent::AtomicFixnum.new
+      @total_output_bytes = Concurrent::AtomicFixnum.new
+      @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
+      @total_response = Concurrent::Map.new
+      @total_response_times = Concurrent::AtomicFixnum.new
+      @total_response_success = Concurrent::AtomicFixnum.new
+  
+    end # def initialize
+
+    def total_response(key)
+      @total_response.get(key) ? @total_response.get(key).value : 0
+    end
+
+    def record_input(size)
+      @total_input_events.increment()
+      @total_input_bytes.update { |v| v + size }
+    end # def record_input
+
+    def record_log_process()
+      @total_log_lines.increment()
+    end # def record_log_process
+
+    def record_metrics_process(dps)
+      @total_metrics_datapoints.update { |v| v + dps }
+    end # def record_metrics_process
+
+    def record_enque(size)
+      @total_enque_times.increment()
+      @total_enque_bytes.update { |v| v + size }
+    end # def record_enque
+
+    def record_deque(size)
+      @total_deque_times.increment()
+      @total_deque_bytes.update { |v| v + size }
+    end # def record_deque
+
+    def record_request(size, size_compressed)
+      @total_output_requests.increment()
+      @total_output_bytes.update { |v| v + size }
+      @total_output_bytes_compressed.update { |v| v + size_compressed }
+    end # def record_request
+
+    def record_response_success(code)
+      atomic_map_increase(@total_response, code.to_s)
+      @total_response_success.increment() if code == 200
+      @total_response_times.increment()
+    end # def record_response_success
+
+    def record_response_failure()
+      atomic_map_increase(@total_response, "failure")
+    end # def record_response_failure
+
+    def atomic_map_increase(map, key)
+      number = map.get(key)
+      if number.nil?
+        newNumber = Concurrent::AtomicFixnum.new
+        number = map.put_if_absent(key, newNumber)
+        if number.nil?
+          number = newNumber
+        end
+      end
+      number.increment()
+    end # def atomic_map_increase
+
+  end
+end; end; end

data/logstash-output-sumologic.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+  s.name          = 'hshek-logstash-output-sumologic'
+  s.version       = '0.0.2'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Deliever the log to Sumo Logic cloud service.'
+  s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
+  s.authors       = ['Sumo Logic']
+  s.email         = 'collection@sumologic.com '
+  s.homepage      = 'https://github.com/SumoLogic/logstash-output-sumologic'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '~> 6.0'
+  
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/outputs/sumologic/compressor_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+
+describe LogStash::Outputs::SumoLogic::Compressor do
+
+  context "compress (deflate)" do
+    let(:compressor) {
+      LogStash::Outputs::SumoLogic::Compressor.new("compress" => true, "compress_encoding" => "deflate")
+    }
+    specify {
+      expect(compressor.compress("abcde").bytesize).to eq(13)
+      expect(compressor.compress("aaaaa").bytesize).to eq(11)
+    }
+  end # context
+  
+  context "compress (gzip)" do
+    let(:compressor) {
+      LogStash::Outputs::SumoLogic::Compressor.new("compress" => true, "compress_encoding" => "gzip")
+    }
+    specify {
+      expect(compressor.compress("abcde").bytesize).to eq(25)
+      expect(compressor.compress("aaaaa").bytesize).to eq(23)
+    }
+  end # context
+
+end # describe

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -0,0 +1,244 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+
+describe LogStash::Outputs::SumoLogic::HeaderBuilder do
+
+  result = {}
+  event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
+
+  before :each do
+    result = builder.build(event)
+  end
+
+  context "should build headers by default" do
+
+    let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
+
+    specify {
+      expected = {
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash",
+        "Content-Type" => "text/plain"
+      }
+      expect(result).to eq(expected)
+    }
+
+  end # context
+
+  context "should override source_category" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category")
+    }
+
+  end # context
+
+  context "should override source_category with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category bar")
+    }
+
+  end # context
+
+  context "should override source_name" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name")
+    }
+
+  end # context
+
+  context "should override source_name with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name bar")
+    }
+
+  end # context
+
+  context "should override source_host" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host")
+    }
+
+  end # context
+
+  context "should override source_host with template" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host %{foo}")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host bar")
+    }
+
+  end # context
+
+  context "should hornor extra_headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar"
+        })
+    }
+
+    specify {
+      expect(result.count).to eq(6)
+      expect(result["foo"]).to eq("bar")
+    }
+
+  end # context
+
+  context "should hornor extra_headers but never overwrite pre-defined headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar",
+          "X-Sumo-Client" => "a",
+          "X-Sumo-Name" => "b",
+          "X-Sumo-Host" => "c",
+          "X-Sumo-Category" => "d",
+          "Content-Type" => "e"
+      })
+    }
+
+    specify {
+      expected = {
+        "foo" => "bar",
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash",
+        "Content-Type" => "text/plain"
+      }
+      expect(result).to eq(expected)
+    }
+
+  end # context
+
+  context "should set content type correctly for log payload" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("text/plain")
+    }
+
+  end # context
+
+  context "should set content type correctly for metrics payload (CarbonV2, default)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "fields_as_metrics" => true)
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("application/vnd.sumologic.carbon2")
+    }
+
+  end # context
+
+  context "should set content type correctly for metrics payload (Graphite)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "metrics_format" => "graphite",
+        "fields_as_metrics" => true)
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("application/vnd.sumologic.graphite")
+    }
+
+  end # context
+
+  context "should set content encoding correctly for uncompressed payload" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to be_nil
+    }
+
+  end # context
+
+  context "should set content encoding correctly for compressed payload (deflate, default)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true)
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to eq("deflate")
+    }
+
+  end # context
+
+  context "should set content encoding correctly for compressed payload (gzip)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true, "compress_encoding" => "gzip")
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to eq("gzip")
+    }
+
+  end # context
+
+end # describe