logstash-output-sumologic 1.1.4 → 1.1.9

data/lib/logstash/outputs/sumologic/sender.rb

@@ -0,0 +1,167 @@
+# encoding: utf-8
+require "net/https"
+require "socket"
+require "thread"
+require "uri"
+require "logstash/outputs/sumologic/common"
+require "logstash/outputs/sumologic/compressor"
+require "logstash/outputs/sumologic/header_builder"
+require "logstash/outputs/sumologic/statistics"
+require "logstash/outputs/sumologic/message_queue"
+
+module LogStash; module Outputs; class SumoLogic;
+  class Sender
+
+    include LogStash::Outputs::SumoLogic::Common
+    STOP_TAG = "PLUGIN STOPPED"
+
+    def initialize(client, queue, stats, config)
+      @client = client
+      @queue = queue
+      @stats = stats
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @url = config["url"]
+      @sender_max = (config["sender_max"] ||= 1) < 1 ? 1 : config["sender_max"]
+      @sleep_before_requeue = config["sleep_before_requeue"] ||= 30
+      @stats_enabled = config["stats_enabled"] ||= false
+
+      @tokens = SizedQueue.new(@sender_max)
+      @sender_max.times { |t| @tokens << t }
+
+      @header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
+      @headers = @header_builder.build()
+      @stats_headers = @header_builder.build_stats()
+      @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
+
+    end # def initialize
+
+    def start()
+      @stopping.make_false()
+      @sender_t = Thread.new {
+        while @stopping.false?
+          content = @queue.deq()
+          send_request(content)
+        end # while
+        @queue.drain().map { |content| 
+          send_request(content)
+        }
+        log_info "waiting messages sent out..."
+        while @tokens.size < @sender_max
+          sleep 1
+        end # while
+      }
+    end # def start_sender
+
+    def stop()
+      log_info "shutting down sender..."
+      @stopping.make_true()
+      @queue.enq(STOP_TAG)
+      @sender_t.join
+      log_info "sender is fully shutted down"
+    end # def stop_sender
+    
+    def connect()
+      uri = URI.parse(@url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = @url.downcase().start_with?("https")
+      request = Net::HTTP::Get.new(uri.request_uri)
+      begin
+        res = http.request(request)
+        if res.code.to_i != 200
+          log_err(
+            "Server rejected the request",
+            :url => @url,
+            :code => res.code
+          )
+          false
+        else
+          log_dbg(
+            "Server accepted the request",
+            :url => @url
+          )
+          true
+        end
+      rescue Exception => ex
+        log_err(
+          "Cannot connect to given url",
+          :url => @url,
+          :exception => ex
+        )
+        false
+      end
+    end # def connect
+    
+    private
+
+    def send_request(content)
+      if content == STOP_TAG
+        log_dbg "STOP_TAG is received."
+        return
+      end
+      
+      token = @tokens.pop()
+
+      if @stats_enabled && content.start_with?(STATS_TAG)
+        body = @compressor.compress(content[STATS_TAG.length..-1])
+        headers = @stats_headers
+      else
+        body = @compressor.compress(content)
+        headers = @headers
+      end
+  
+      request = @client.send(:background).send(:post, @url, :body => body, :headers => headers)
+      
+      request.on_complete do
+        @tokens << token
+      end
+  
+      request.on_success do |response|
+        @stats.record_response_success(response.code)
+        if response.code < 200 || response.code > 299
+          log_err(
+            "HTTP request rejected(#{response.code})",
+            :token => token,
+            :code => response.code,
+            :headers => headers,
+            :contet => content[0..20]
+          )
+          if response.code == 429 || response.code == 503 || response.code == 504
+            requeue_message(content)
+          end
+        else
+          log_dbg(
+            "HTTP request accepted",
+            :token => token,
+            :code => response.code)
+        end
+      end
+  
+      request.on_failure do |exception|
+        @stats.record_response_failure()
+        log_err(
+          "Error in network transmission",
+          :token => token,
+          :message => exception.message,
+          :class => exception.class.name,
+          :backtrace => exception.backtrace
+        )
+        requeue_message(content)
+      end      
+
+      @stats.record_request(content.bytesize, body.bytesize)
+      request.call
+    end # def send_request
+
+    def requeue_message(content)
+      if @stopping.false? && @sleep_before_requeue >= 0
+        log_warn(
+          "requeue message",
+          :after => @sleep_before_requeue,
+          :content => content[0..20])
+        Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
+        @queue.enq(content)
+      end
+    end # def reque_message
+
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/statistics.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "logstash/outputs/sumologic/common"
+
+module LogStash; module Outputs; class SumoLogic;
+  class Statistics
+
+    include LogStash::Outputs::SumoLogic::Common
+
+    attr_reader :initialize_time
+    attr_reader :total_input_events
+    attr_reader :total_input_bytes
+    attr_reader :total_metrics_datapoints
+    attr_reader :total_log_lines
+    attr_reader :current_pile_items
+    attr_reader :current_pile_bytes
+    attr_reader :total_enque_times
+    attr_reader :total_enque_bytes
+    attr_reader :total_deque_times
+    attr_reader :total_deque_bytes
+    attr_reader :current_queue_items
+    attr_reader :current_queue_bytes
+    attr_reader :total_output_requests
+    attr_reader :total_output_bytes
+    attr_reader :total_output_bytes_compressed 
+    attr_reader :total_response
+    attr_reader :total_response_times
+    attr_reader :total_response_success
+
+    def initialize()
+      @initialize_time = Time.now()
+      @total_input_events = Concurrent::AtomicFixnum.new
+      @total_input_bytes = Concurrent::AtomicFixnum.new
+      @total_metrics_datapoints = Concurrent::AtomicFixnum.new
+      @total_log_lines = Concurrent::AtomicFixnum.new
+      @current_pile_items = Concurrent::AtomicFixnum.new
+      @current_pile_bytes = Concurrent::AtomicFixnum.new
+      @total_enque_times = Concurrent::AtomicFixnum.new
+      @total_enque_bytes = Concurrent::AtomicFixnum.new
+      @total_deque_times = Concurrent::AtomicFixnum.new
+      @total_deque_bytes = Concurrent::AtomicFixnum.new
+      @current_queue_items = Concurrent::AtomicFixnum.new
+      @current_queue_bytes = Concurrent::AtomicFixnum.new
+      @total_output_requests = Concurrent::AtomicFixnum.new
+      @total_output_bytes = Concurrent::AtomicFixnum.new
+      @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
+      @total_response = Concurrent::Map.new
+      @total_response_times = Concurrent::AtomicFixnum.new
+      @total_response_success = Concurrent::AtomicFixnum.new
+  
+    end # def initialize
+
+    def total_response(key)
+      @total_response.get(key) ? @total_response.get(key).value : 0
+    end
+
+    def record_multi_input(events, bytesize)
+      @total_input_events.update { |v| v + events }
+      @total_input_bytes.update { |v| v + bytesize }
+    end # def record_multi_input
+
+    def record_input(entry)
+      @total_input_events.increment()
+      @total_input_bytes.update { |v| v + entry.bytesize }
+      @current_pile_items.increment()
+      @current_pile_bytes.update { |v| v + entry.bytesize }
+    end # def record_input
+
+    def record_log_process()
+      @total_log_lines.increment()
+    end # def record_log_process
+
+    def record_metrics_process(dps)
+      @total_metrics_datapoints.update { |v| v + dps }
+    end # def record_metrics_process
+
+    def record_clear_pile()
+      @current_pile_items.value= 0
+      @current_pile_bytes.value= 0
+    end # def record_pile_clear
+
+    def record_enque(payload)
+      @total_enque_times.increment()
+      @total_enque_bytes.update { |v| v + payload.bytesize }
+      @current_queue_items.increment()
+      @current_queue_bytes.update { |v| v + payload.bytesize }
+    end # def record_enque
+
+    def record_deque(payload)
+      @total_deque_times.increment()
+      @total_deque_bytes.update { |v| v + payload.bytesize }
+      @current_queue_items.decrement()
+      @current_queue_bytes.update { |v| v - payload.bytesize }
+    end # def record_deque
+
+    def record_request(size, size_compressed)
+      @total_output_requests.increment()
+      @total_output_bytes.update { |v| v + size }
+      @total_output_bytes_compressed.update { |v| v + size_compressed }
+    end # def record_request
+
+    def record_response_success(code)
+      atomic_map_increase(@total_response, code.to_s)
+      @total_response_success.increment() if code == 200
+      @total_response_times.increment()
+    end # def record_response_success
+
+    def record_response_failure()
+      atomic_map_increase(@total_response, "failure")
+    end # def record_response_failure
+
+    def atomic_map_increase(map, key)
+      number = map.get(key)
+      if number.nil?
+        newNumber = Concurrent::AtomicFixnum.new
+        number = map.put_if_absent(key, newNumber)
+        if number.nil?
+          number = newNumber
+        end
+      end
+      number.increment()
+    end # def atomic_map_increase
+
+  end
+end; end; end

data/logstash-output-sumologic.gemspec

@@ -1,25 +1,27 @@
 Gem::Specification.new do |s|
-  s.name          = "logstash-output-sumologic"
-  s.version       = "1.1.4"
-  s.licenses      = ["Apache-2.0"]
-  s.summary       = "Deliever the log to Sumo Logic cloud service."
-  s.description   = "This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc."
-  s.authors       = ["Sumo Logic"]
-  s.email         = "collection@sumologic.com "
-  s.homepage      = "https://github.com/SumoLogic/logstash-output-sumologic"
-  s.require_paths = ["lib"]
+  s.name          = 'logstash-output-sumologic'
+  s.version       = '1.1.9'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Deliever the log to Sumo Logic cloud service.'
+  s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
+  s.authors       = ['Sumo Logic']
+  s.email         = 'collection@sumologic.com '
+  s.homepage      = 'https://github.com/SumoLogic/logstash-output-sumologic'
+  s.require_paths = ['lib']
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","vendor/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT"]
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
    # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_runtime_dependency "logstash-codec-plain", ">= 0"
-  s.add_runtime_dependency 'logstash-mixin-http_client', ">= 6.0.0", "< 7.0.0"
-  s.add_development_dependency "logstash-devutils", "~> 1.3", ">= 1.3.1"
+  s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-mixin-http_client'
+  
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/sumologic/compressor_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+
+describe LogStash::Outputs::SumoLogic::Compressor do
+
+  context "compress (deflate)" do
+    let(:compressor) {
+      LogStash::Outputs::SumoLogic::Compressor.new("compress" => true, "compress_encoding" => "deflate")
+    }
+    specify {
+      expect(compressor.compress("abcde").bytesize).to eq(13)
+      expect(compressor.compress("aaaaa").bytesize).to eq(11)
+    }
+  end # context
+  
+  context "compress (gzip)" do
+    let(:compressor) {
+      LogStash::Outputs::SumoLogic::Compressor.new("compress" => true, "compress_encoding" => "gzip")
+    }
+    specify {
+      expect(compressor.compress("abcde").bytesize).to eq(25)
+      expect(compressor.compress("aaaaa").bytesize).to eq(23)
+    }
+  end # context
+
+end # describe

data/spec/outputs/sumologic/header_builder_spec.rb

@@ -0,0 +1,197 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/sumologic"
+
+describe LogStash::Outputs::SumoLogic::HeaderBuilder do
+
+  result = {}
+
+  before :each do
+    result = builder.build()
+  end
+
+  context "should build headers by default" do
+    let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
+
+    specify {
+      expected = {
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash",
+        "Content-Type" => "text/plain"
+      }
+      expect(result).to eq(expected)
+    }
+
+  end # context
+
+  context "should override source_category" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_category" => "my source category")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Category"]).to eq("my source category")
+    }
+
+  end # context
+
+  context "should override source_name" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_name" => "my source name")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Name"]).to eq("my source name")
+    }
+
+  end # context
+
+  context "should override source_host" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "source_host" => "my source host")
+    }
+
+    specify {
+      expect(result.count).to eq(5)
+      expect(result["X-Sumo-Host"]).to eq("my source host")
+    }
+
+  end # context
+
+  context "should hornor extra_headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar"
+        })
+    }
+
+    specify {
+      expect(result.count).to eq(6)
+      expect(result["foo"]).to eq("bar")
+    }
+
+  end # context
+
+  context "should hornor extra_headers but never overwrite pre-defined headers" do
+    
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "extra_headers" => {
+          "foo" => "bar",
+          "X-Sumo-Client" => "a",
+          "X-Sumo-Name" => "b",
+          "X-Sumo-Host" => "c",
+          "X-Sumo-Category" => "d",
+          "Content-Type" => "e"
+      })
+    }
+
+    specify {
+      expected = {
+        "foo" => "bar",
+        "X-Sumo-Client" => "logstash-output-sumologic",
+        "X-Sumo-Name" => "logstash-output-sumologic",
+        "X-Sumo-Host" => Socket.gethostname,
+        "X-Sumo-Category" => "Logstash",
+        "Content-Type" => "text/plain"
+      }
+      expect(result).to eq(expected)
+    }
+
+  end # context
+
+  context "should set content type correctly for log payload" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("text/plain")
+    }
+
+  end # context
+
+  context "should set content type correctly for metrics payload (CarbonV2, default)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "fields_as_metrics" => true)
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("application/vnd.sumologic.carbon2")
+    }
+
+  end # context
+
+  context "should set content type correctly for metrics payload (Graphite)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new(
+        "url" => "http://localhost/1234",
+        "metrics_format" => "graphite",
+        "fields_as_metrics" => true)
+    }
+
+    specify {
+      expect(result["Content-Type"]).to eq("application/vnd.sumologic.graphite")
+    }
+
+  end # context
+
+  context "should set content encoding correctly for uncompressed payload" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to be_nil
+    }
+
+  end # context
+
+  context "should set content encoding correctly for compressed payload (deflate, default)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true)
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to eq("deflate")
+    }
+
+  end # context
+
+  context "should set content encoding correctly for compressed payload (gzip)" do
+
+    let(:builder) {
+      LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true, "compress_encoding" => "gzip")
+    }
+
+    specify {
+      expect(result["Content-Encoding"]).to eq("gzip")
+    }
+
+  end # context
+
+end # describe