hshek-logstash-output-sumologic 0.0.2

data/lib/logstash/outputs/sumologic/batch.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Batch
+    
+    attr_accessor :headers, :payload
+  
+    def initialize(headers, payload)
+        @headers, @payload = headers, payload
+    end
+  
+  end
+end; end; end;

data/lib/logstash/outputs/sumologic/common.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+module LogStash; module Outputs; class SumoLogic;
+  module Common
+
+    require "date"
+
+    # global constants
+    DEFAULT_LOG_FORMAT = "%{@timestamp} %{host} %{message}"
+    METRICS_NAME_PLACEHOLDER = "*"
+    GRAPHITE = "graphite"
+    CARBON2 = "carbon2"
+    DEFLATE = "deflate"
+    GZIP = "gzip"
+    STATS_TAG = "STATS_TAG"
+    STOP_TAG = "PLUGIN STOPPED"
+
+    CONTENT_TYPE = "Content-Type"
+    CONTENT_TYPE_LOG = "text/plain"
+    CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
+    CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
+    CONTENT_ENCODING = "Content-Encoding"
+
+    CATEGORY_HEADER = "X-Sumo-Category"
+    CATEGORY_HEADER_DEFAULT = "Logstash"
+    HOST_HEADER = "X-Sumo-Host"
+    NAME_HEADER = "X-Sumo-Name"
+    NAME_HEADER_DEFAULT = "logstash-output-sumologic"
+
+    CLIENT_HEADER = "X-Sumo-Client"
+    CLIENT_HEADER_VALUE = "logstash-output-sumologic"
+
+    # for debugging test
+    LOG_TO_CONSOLE = false
+    @@logger = nil
+
+    def set_logger(logger)
+      @@logger = logger
+    end
+
+    def log_info(message, *opts)
+      if LOG_TO_CONSOLE
+        puts "[INFO:#{DateTime::now}]#{message} #{opts.to_s}"
+      else
+        @@logger && @@logger.info(message, *opts)
+      end
+    end # def log_info
+
+    def log_warn(message, *opts)
+      if LOG_TO_CONSOLE
+        puts "\e[33m[WARN:#{DateTime::now}]#{message} #{opts.to_s}\e[0m"
+      else
+        @@logger && @@logger.warn(message, *opts)
+      end
+    end # def log_warn
+
+    def log_err(message, *opts)
+      if LOG_TO_CONSOLE
+        puts "\e[31m[ERR :#{DateTime::now}]#{message} #{opts.to_s}\e[0m"
+      else
+        @@logger && @@logger.error(message, *opts)
+      end
+    end # def log_err
+
+    def log_dbg(message, *opts)
+      if LOG_TO_CONSOLE
+        puts "\e[36m[DBG :#{DateTime::now}]#{message} #{opts.to_s}\e[0m"
+      else
+        @@logger && @@logger.debug(message, *opts)
+      end
+    end # def log_dbg
+
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/compressor.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Compressor
+
+    require "stringio"
+    require "zlib"
+    require "logstash/outputs/sumologic/common"
+    include LogStash::Outputs::SumoLogic::Common
+
+    def initialize(config)
+      @compress = config["compress"]
+      @compress_encoding = (config["compress_encoding"] ||= DEFLATE).downcase
+    end # def initialize
+
+    def compress(content)
+      if @compress
+        if @compress_encoding == GZIP
+          result = gzip(content)
+          result.bytes.to_a.pack("c*")
+        else
+          Zlib::Deflate.deflate(content)
+        end
+      else
+        content
+      end
+    end # def compress
+    
+    def gzip(content)
+      stream = StringIO.new("w")
+      stream.set_encoding("ASCII")
+      gz = Zlib::GzipWriter.new(stream)
+      gz.write(content)
+      gz.close
+      stream.string.bytes.to_a.pack("c*")
+    end # def gzip
+  
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/header_builder.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class HeaderBuilder
+
+    require "socket"
+    require "logstash/outputs/sumologic/common"
+    include LogStash::Outputs::SumoLogic::Common
+
+    def initialize(config)
+      
+      @extra_headers = config["extra_headers"] ||= {}
+      @source_category = config["source_category"] ||= CATEGORY_HEADER_DEFAULT
+      @source_host = config["source_host"] ||= Socket.gethostname
+      @source_name = config["source_name"] ||= NAME_HEADER_DEFAULT
+      @metrics = config["metrics"]
+      @fields_as_metrics = config["fields_as_metrics"]
+      @metrics_format = (config["metrics_format"] ||= CARBON2).downcase
+      @compress = config["compress"]
+      @compress_encoding = config["compress_encoding"]
+
+    end # def initialize
+    
+    def build(event)
+      headers = Hash.new
+      headers.merge!(@extra_headers)
+      headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
+      headers[CATEGORY_HEADER] = event.sprintf(@source_category) unless @source_category.blank?
+      headers[HOST_HEADER] = event.sprintf(@source_host) unless @source_host.blank?
+      headers[NAME_HEADER] = event.sprintf(@source_name) unless @source_name.blank?
+      append_content_header(headers)
+      append_compress_header(headers)
+      headers
+    end # def build
+
+    private
+    def append_content_header(headers)
+      contentType = CONTENT_TYPE_LOG
+      if @metrics || @fields_as_metrics
+        contentType = (@metrics_format == GRAPHITE) ? CONTENT_TYPE_GRAPHITE : CONTENT_TYPE_CARBON2
+      end
+      headers[CONTENT_TYPE] = contentType
+    end # def append_content_header
+
+    def append_compress_header(headers)
+      if @compress
+        headers[CONTENT_ENCODING] = (@compress_encoding == GZIP) ? GZIP : DEFLATE
+      end
+    end # append_compress_header
+
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/message_queue.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+module LogStash; module Outputs; class SumoLogic;
+  class MessageQueue
+
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    include LogStash::Outputs::SumoLogic::Common
+
+    def initialize(stats, config)
+      @queue_max = (config["queue_max"] ||= 1) < 1 ? 1 : config["queue_max"]
+      @queue = SizedQueue::new(@queue_max)
+      log_info("initialize memory queue", :max => @queue_max)
+      @queue_bytesize = Concurrent::AtomicFixnum.new
+      @stats = stats
+    end # def initialize
+
+    def enq(batch)
+      batch_size = batch.payload.bytesize
+      if (batch_size > 0)
+        @queue.enq(batch)
+        @stats.record_enque(batch_size)
+        @queue_bytesize.update { |v| v + batch_size }
+        log_dbg("enqueue",
+          :objects_in_queue => size,
+          :bytes_in_queue => @queue_bytesize,
+          :size => batch_size)
+        end
+    end # def enq
+
+    def deq()
+      batch = @queue.deq()
+      batch_size = batch.payload.bytesize
+      @stats.record_deque(batch_size)
+      @queue_bytesize.update { |v| v - batch_size }
+      log_dbg("dequeue",
+        :objects_in_queue => size,
+        :bytes_in_queue => @queue_bytesize,
+        :size => batch_size)
+      batch
+    end # def deq
+
+    def drain()
+      @queue.size.times.map {
+        deq()
+      }
+    end # def drain
+
+    def size()
+      @queue.size()
+    end # size
+
+    def bytesize()
+      @queue_bytesize.value
+    end # bytesize
+    
+  end
+end; end; end

data/lib/logstash/outputs/sumologic/monitor.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class Monitor
+
+    require "logstash/outputs/sumologic/common"
+    require "logstash/outputs/sumologic/statistics"
+    require "logstash/outputs/sumologic/message_queue"
+    include LogStash::Outputs::SumoLogic::Common
+
+    attr_reader :is_pile
+
+    def initialize(queue, stats, config)
+      @queue = queue
+      @stats = stats
+      @stopping = Concurrent::AtomicBoolean.new(false)
+
+      @enabled = config["stats_enabled"] ||= false
+      @interval = config["stats_interval"] ||= 60
+      @interval = @interval < 0 ? 0 : @interval
+    end # initialize
+
+    def start()
+      log_info("starting monitor...", :interval => @interval)
+      @stopping.make_false()
+      if (@enabled)
+        @monitor_t = Thread.new { 
+          while @stopping.false?
+            Stud.stoppable_sleep(@interval) { @stopping.true? }
+            if @stats.total_input_events.value > 0
+              @queue.enq(build_stats_payload())
+            end
+          end # while
+        }
+      end # if
+    end # def start
+
+    def stop()
+      @stopping.make_true()
+      if (@enabled)
+        log_info("shutting down monitor...")
+        @monitor_t.join
+        log_info("monitor is fully shutted down")
+      end
+    end # def stop
+
+    def build_stats_payload()
+      timestamp = Time.now().to_i
+      
+      counters = [
+        "total_input_events", 
+        "total_input_bytes", 
+        "total_metrics_datapoints", 
+        "total_log_lines", 
+        "total_output_requests",
+        "total_output_bytes",
+        "total_output_bytes_compressed",
+        "total_response_times",
+        "total_response_success"
+      ].map { |key|
+        value = @stats.send(key).value
+        log_dbg("stats",
+          :key => key,
+          :value => value)
+        build_metric_line(key, value, timestamp)
+      }.join($/)
+
+      "#{STATS_TAG}#{counters}"
+    end # def build_stats_payload
+
+    def build_metric_line(key, value, timestamp)
+      "metric=#{key} interval=#{@interval}  category=monitor #{value} #{timestamp}"
+    end # def build_metric_line
+
+  end
+end; end; end        

data/lib/logstash/outputs/sumologic/payload_builder.rb

@@ -0,0 +1,159 @@
+# encoding: utf-8
+
+module LogStash; module Outputs; class SumoLogic;
+  class PayloadBuilder
+    
+    require "logstash/json"
+    require "logstash/event"
+    require "logstash/outputs/sumologic/common"
+    include LogStash::Outputs::SumoLogic::Common
+
+    TIMESTAMP_FIELD = "@timestamp"
+    METRICS_NAME_TAG = "metric"
+    JSON_PLACEHOLDER = "%{@json}"
+    ALWAYS_EXCLUDED = [ "@timestamp", "@version" ]
+    
+    def initialize(stats, config)
+      @stats = stats
+      
+      @format = config["format"] ||= DEFAULT_LOG_FORMAT
+      @json_mapping = config["json_mapping"]
+
+      @metrics = config["metrics"]
+      @metrics_name = config["metrics_name"]
+      @fields_as_metrics = config["fields_as_metrics"]
+      @metrics_format = (config["metrics_format"] ||= CARBON2).downcase
+      @intrinsic_tags = config["intrinsic_tags"] ||= {}
+      @meta_tags = config["meta_tags"] ||= {}
+      @fields_include = config["fields_include"] ||= []
+      @fields_exclude = config["fields_exclude"] ||= []
+
+    end # def initialize
+
+    def build(event)
+      payload = if @metrics || @fields_as_metrics
+        build_metrics_payload(event)
+      else
+        build_log_payload(event)
+      end
+      payload
+    end # def build
+  
+    private
+
+    def build_log_payload(event)
+      @stats.record_log_process()
+      apply_template(@format, event)
+    end # def event2log
+  
+    def build_metrics_payload(event)
+      timestamp = event.get(TIMESTAMP_FIELD).to_i
+      source = if @fields_as_metrics
+        event_as_metrics(event)
+      else
+        expand_hash(@metrics, event)
+      end
+      lines = source.flat_map { |key, value|
+        get_single_line(event, key, value, timestamp)
+      }.reject(&:nil?)
+      @stats.record_metrics_process(lines.size)
+      lines.join($/)
+    end # def event2metrics
+  
+    def event_as_metrics(event)
+      hash = event2hash(event)
+      acc = {}
+      hash.keys.each do |field|
+        value = hash[field]
+        dotify(acc, field, value, nil)
+      end
+      acc
+    end # def event_as_metrics
+  
+    def get_single_line(event, key, value, timestamp)
+      full = get_metrics_name(event, key)
+      if !ALWAYS_EXCLUDED.include?(full) &&  \
+        (@fields_include.empty? || @fields_include.any? { |regexp| full.match(regexp) }) && \
+        !(@fields_exclude.any? {|regexp| full.match(regexp)}) && \
+        is_number?(value)
+        if @metrics_format == GRAPHITE
+          "#{full} #{value} #{timestamp}" 
+        else
+          @intrinsic_tags[METRICS_NAME_TAG] = full
+          "#{hash2line(@intrinsic_tags, event)} #{hash2line(@meta_tags, event)}#{value} #{timestamp}"
+        end
+      end
+    end # def get_single_line
+  
+    def dotify(acc, key, value, prefix)
+      pk = prefix ? "#{prefix}.#{key}" : key.to_s
+      if value.is_a?(Hash)
+        value.each do |k, v|
+          dotify(acc, k, v, pk)
+        end
+      elsif value.is_a?(Array)
+        value.each_with_index.map { |v, i|
+          dotify(acc, i.to_s, v, pk)
+        }
+      else
+        acc[pk] = value
+      end
+    end # def dotify
+  
+    def event2hash(event)
+      if @json_mapping
+        @json_mapping.reduce({}) do |acc, kv|
+          k, v = kv
+          acc[k] = event.sprintf(v)
+          acc
+        end
+      else
+        event.to_hash
+      end
+    end # def map_event
+  
+    def is_number?(me)
+      me.to_f.to_s == me.to_s || me.to_i.to_s == me.to_s
+    end # def is_number?
+  
+    def expand_hash(hash, event)
+      hash.reduce({}) do |acc, kv|
+        k, v = kv
+        exp_k = apply_template(k, event)
+        exp_v = apply_template(v, event)
+        acc[exp_k] = exp_v
+        acc
+      end
+    end # def expand_hash
+    
+    def apply_template(template, event)
+      if template == JSON_PLACEHOLDER
+        hash = event2hash(event)
+        LogStash::Json.dump(hash)
+      elsif template.include? JSON_PLACEHOLDER
+        result = event.sprintf(template)
+        hash = event2hash(event)
+        dump = LogStash::Json.dump(hash)
+        result.gsub(JSON_PLACEHOLDER) { dump }
+      else
+        event.sprintf(template)
+      end
+    end # def expand
+    
+    def get_metrics_name(event, name)
+      name = @metrics_name.gsub(METRICS_NAME_PLACEHOLDER) { name } if @metrics_name
+      event.sprintf(name)
+    end # def get_metrics_name
+  
+    def hash2line(hash, event)
+      if (hash.is_a?(Hash) && !hash.empty?)
+        expand_hash(hash, event).flat_map { |k, v|
+          "#{k}=#{v} "
+        }.join()
+      else
+        ""
+      end
+    end # def hash2line
+    
+  end
+end; end; end