logstash-lib 1.3.2

data/lib/logstash/codecs/json_spooler.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/spool"
+
+# This is the base class for logstash codecs.
+class LogStash::Codecs::JsonSpooler < LogStash::Codecs::Spool
+  config_name "json_spooler"
+  milestone 1
+
+  public
+  def decode(data)
+    super(JSON.parse(data.force_encoding("UTF-8"))) do |event|
+      yield event
+    end
+  end # def decode
+
+  public
+  def encode(data)
+    super(data)
+  end # def encode
+
+end # class LogStash::Codecs::Json

data/lib/logstash/codecs/line.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/util/charset"
+
+# Line-oriented text data.
+#
+# Decoding behavior: Only whole line events will be emitted.
+#
+# Encoding behavior: Each event will be emitted with a trailing newline.
+class LogStash::Codecs::Line < LogStash::Codecs::Base
+  config_name "line"
+  milestone 3
+
+  # Set the desired text format for encoding.
+  config :format, :validate => :string
+
+  # The character encoding used in this input. Examples include "UTF-8"
+  # and "cp1252"
+  #
+  # This setting is useful if your log files are in Latin-1 (aka cp1252)
+  # or in another character set other than UTF-8.
+  #
+  # This only affects "plain" format logs since json is UTF-8 already.
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def register
+    require "logstash/util/buftok"
+    @buffer = FileWatch::BufferedTokenizer.new
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+  
+  public
+  def decode(data)
+    @buffer.extract(data).each do |line|
+      yield LogStash::Event.new("message" => @converter.convert(line))
+    end
+  end # def decode
+
+  public
+  def flush(&block)
+    remainder = @buffer.flush
+    if !remainder.empty?
+      block.call(LogStash::Event.new({"message" => remainder}))
+    end
+  end
+
+  public
+  def encode(data)
+    if data.is_a? LogStash::Event and @format
+      @on_event.call(data.sprintf(@format) + "\n")
+    else
+      @on_event.call(data.to_s + "\n")
+    end
+  end # def encode
+
+end # class LogStash::Codecs::Plain

data/lib/logstash/codecs/msgpack.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::Msgpack < LogStash::Codecs::Base
+  config_name "msgpack"
+
+  milestone 1
+
+  config :format, :validate => :string, :default => nil
+
+  public
+  def register
+    require "msgpack"
+  end
+
+  public
+  def decode(data)
+    begin
+      # Msgpack does not care about UTF-8
+      event = LogStash::Event.new(MessagePack.unpack(data))
+      event["@timestamp"] = Time.at(event["@timestamp"]).utc if event["@timestamp"].is_a? Float
+      event["tags"] ||= []
+      if @format
+        event["message"] ||= event.sprintf(@format)
+      end
+    rescue => e
+      # Treat as plain text and try to do the best we can with it?
+      @logger.warn("Trouble parsing msgpack input, falling back to plain text",
+                   :input => data, :exception => e)
+      event["message"] = data
+      event["tags"] ||= []
+      event["tags"] << "_msgpackparsefailure"
+    end
+    yield event
+  end # def decode
+
+  public
+  def encode(event)
+    event["@timestamp"] = event["@timestamp"].to_f
+    @on_event.call event.to_hash.to_msgpack
+  end # def encode
+
+end # class LogStash::Codecs::Msgpack

data/lib/logstash/codecs/multiline.rb

@@ -0,0 +1,189 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+# The multiline codec is for taking line-oriented text and merging them into a
+# single event.
+#
+# The original goal of this codec was to allow joining of multi-line messages
+# from files into a single event. For example - joining java exception and
+# stacktrace messages into a single event.
+#
+# The config looks like this:
+#
+#     input {
+#       stdin {
+#         codec => multiline {
+#           pattern => "pattern, a regexp"
+#           negate => true or false
+#           what => "previous" or "next"
+#         }
+#       }
+#     }
+# 
+# The 'pattern' should match what you believe to be an indicator that the field
+# is part of a multi-line event.
+#
+# The 'what' must be "previous" or "next" and indicates the relation
+# to the multi-line event.
+#
+# The 'negate' can be "true" or "false" (defaults false). If true, a 
+# message not matching the pattern will constitute a match of the multiline
+# filter and the what will be applied. (vice-versa is also true)
+#
+# For example, java stack traces are multiline and usually have the message
+# starting at the far-left, then each subsequent line indented. Do this:
+# 
+#     input {
+#       stdin {
+#         codec => multiline {
+#           pattern => "^\s"
+#           what => "previous"
+#         }
+#       }
+#     }
+#     
+# This says that any line starting with whitespace belongs to the previous line.
+#
+# Another example is to merge lines not starting with a date up to the previous
+# line..
+#
+#     input {
+#       file {
+#         path => "/var/log/someapp.log"
+#         codec => multiline {
+#           # Grok pattern names are valid! :)
+#           pattern => "^%{TIMESTAMP_ISO8601} "
+#           negate => true
+#           what => previous
+#         }
+#       }
+#     }
+#     
+# This is the base class for logstash codecs.
+class LogStash::Codecs::Multiline < LogStash::Codecs::Base
+  config_name "multiline"
+  milestone 3
+
+  # The regular expression to match
+  config :pattern, :validate => :string, :required => true
+
+  # If the pattern matched, does event belong to the next or previous event?
+  config :what, :validate => ["previous", "next"], :required => true
+
+  # Negate the regexp pattern ('if not matched')
+  config :negate, :validate => :boolean, :default => false
+
+  # logstash ships by default with a bunch of patterns, so you don't
+  # necessarily need to define this yourself unless you are adding additional
+  # patterns.
+  #
+  # Pattern files are plain text with format:
+  #
+  #     NAME PATTERN
+  #
+  # For example:
+  #
+  #     NUMBER \d+
+  config :patterns_dir, :validate => :array, :default => []
+
+  # The character encoding used in this input. Examples include "UTF-8"
+  # and "cp1252"
+  #
+  # This setting is useful if your log files are in Latin-1 (aka cp1252)
+  # or in another character set other than UTF-8.
+  #
+  # This only affects "plain" format logs since json is UTF-8 already.
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  # Tag multiline events with a given tag. This tag will only be added
+  # to events that actually have multiple lines in them.
+  config :multiline_tag, :validate => :string, :default => "multiline"
+
+  public
+  def register
+    require "grok-pure" # rubygem 'jls-grok'
+    # Detect if we are running from a jarfile, pick the right path.
+    patterns_path = []
+    if __FILE__ =~ /file:\/.*\.jar!.*/
+      patterns_path += ["#{File.dirname(__FILE__)}/../../patterns/*"]
+    else
+      patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
+    end
+
+    @grok = Grok.new
+
+    @patterns_dir = patterns_path.to_a + @patterns_dir
+    @patterns_dir.each do |path|
+      # Can't read relative paths from jars, try to normalize away '../'
+      while path =~ /file:\/.*\.jar!.*\/\.\.\//
+        # replace /foo/bar/../baz => /foo/baz
+        path = path.gsub(/[^\/]+\/\.\.\//, "")
+      end
+
+      if File.directory?(path)
+        path = File.join(path, "*")
+      end
+
+      Dir.glob(path).each do |file|
+        @logger.info("Grok loading patterns from file", :path => file)
+        @grok.add_patterns_from_file(file)
+      end
+    end
+
+    @grok.compile(@pattern)
+    @logger.debug("Registered multiline plugin", :type => @type, :config => @config)
+
+    @buffer = []
+    @handler = method("do_#{@what}".to_sym)
+  end # def register
+  
+  public
+  def decode(text, &block)
+    text.force_encoding(@charset)
+    if @charset != "UTF-8"
+      # Convert to UTF-8 if not in that character set.
+      text = text.encode("UTF-8", :invalid => :replace, :undef => :replace)
+    end
+
+    match = @grok.match(text)
+    @logger.debug("Multiline", :pattern => @pattern, :text => text,
+                  :match => !match.nil?, :negate => @negate)
+
+    # Add negate option
+    match = (match and !@negate) || (!match and @negate)
+    @handler.call(text, match, &block)
+  end # def decode
+
+  def buffer(text)
+    @time = Time.now.utc if @buffer.empty?
+    @buffer << text
+  end
+
+  def flush(&block)
+    if @buffer.any?
+      event = LogStash::Event.new("@timestamp" => @time, "message" => @buffer.join("\n"))
+      # Tag multiline events
+      event.tag @multiline_tag if @multiline_tag && @buffer.size > 1
+
+      yield event
+      @buffer = []
+    end
+  end
+
+  def do_next(text, matched, &block)
+    buffer(text)
+    flush(&block) if !matched
+  end
+
+  def do_previous(text, matched, &block)
+    flush(&block) if !matched
+    buffer(text)
+  end
+
+  public
+  def encode(data)
+    # Nothing to do.
+    @on_event.call(data)
+  end # def encode
+
+end # class LogStash::Codecs::Plain

data/lib/logstash/codecs/netflow.rb

@@ -0,0 +1,342 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The "netflow" codec is for decoding Netflow v5/v9 flows.
+class LogStash::Codecs::Netflow < LogStash::Codecs::Base
+  config_name "netflow"
+  milestone 1
+
+  # Netflow v9 template cache TTL (minutes)
+  config :cache_ttl, :validate => :number, :default => 4000
+
+  # Specify into what field you want the Netflow data.
+  config :target, :validate => :string, :default => "netflow"
+
+  # Specify which Netflow versions you will accept.
+  config :versions, :validate => :array, :default => [5, 9]
+
+  public
+  def initialize(params={})
+    super(params)
+    @threadsafe = false
+  end
+
+  public
+  def register
+    require "logstash/codecs/netflow/util"
+    @templates = Vash.new()
+  end # def register
+
+  public
+  def decode(payload, &block)
+    header = Header.read(payload)
+
+    unless @versions.include?(header.version)
+      @logger.warn("Ignoring Netflow version v#{header.version}")
+      return
+    end
+
+    if header.version == 5
+      flowset = Netflow5PDU.read(payload)
+    elsif header.version == 9
+      flowset = Netflow9PDU.read(payload)
+    else
+      @logger.warn("Unsupported Netflow version v#{header.version}")
+      return
+    end
+
+    flowset.records.each do |record|
+      if flowset.version == 5
+        event = LogStash::Event.new
+
+        # FIXME Probably not doing this right WRT JRuby?
+        #
+        # The flowset header gives us the UTC epoch seconds along with
+        # residual nanoseconds so we can set @timestamp to that easily
+        event["@timestamp"] = Time.at(flowset.unix_sec, flowset.unix_nsec / 1000).utc
+        event[@target] = {}
+
+        # Copy some of the pertinent fields in the header to the event
+        ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
+          event[@target][f] = flowset[f]
+        end
+
+        # Create fields in the event from each field in the flow record
+        record.each_pair do |k,v|
+          case k.to_s
+          when /_switched$/
+            # The flow record sets the first and last times to the device
+            # uptime in milliseconds. Given the actual uptime is provided
+            # in the flowset header along with the epoch seconds we can
+            # convert these into absolute times
+            millis = flowset.uptime - v
+            seconds = flowset.unix_sec - (millis / 1000)
+            micros = (flowset.unix_nsec / 1000) - (millis % 1000)
+            if micros < 0
+              seconds--
+              micros += 1000000
+            end
+            # FIXME Again, probably doing this wrong WRT JRuby?
+            event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
+          else
+            event[@target][k.to_s] = v
+          end
+        end
+
+        yield event
+      elsif flowset.version == 9
+        case record.flowset_id
+        when 0
+          # Template flowset
+          record.flowset_data.templates.each do |template|
+            catch (:field) do
+              fields = []
+              template.fields.each do |field|
+                entry = netflow_field_for(field.field_type, field.field_length)
+                if ! entry
+                  throw :field
+                end
+                fields += entry
+              end
+              # We get this far, we have a list of fields
+              #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
+              key = "#{flowset.source_id}|#{template.template_id}"
+              @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+              # Purge any expired templates
+              @templates.cleanup!
+            end
+          end
+        when 1
+          # Options template flowset
+          record.flowset_data.templates.each do |template|
+            catch (:field) do
+              fields = []
+              template.option_fields.each do |field|
+                entry = netflow_field_for(field.field_type, field.field_length)
+                if ! entry
+                  throw :field
+                end
+                fields += entry
+              end
+              # We get this far, we have a list of fields
+              #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
+              key = "#{flowset.source_id}|#{template.template_id}"
+              @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+              # Purge any expired templates
+              @templates.cleanup!
+            end
+          end 
+        when 256..65535
+          # Data flowset
+          #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
+          key = "#{flowset.source_id}|#{record.flowset_id}"
+          template = @templates[key]
+
+          if ! template
+            #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
+            @logger.warn("No matching template for flow id #{record.flowset_id}")
+            next
+          end
+
+          length = record.flowset_length - 4
+
+          # Template shouldn't be longer than the record and there should
+          # be at most 3 padding bytes
+          if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
+            @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length) 
+            next
+          end
+
+          array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
+
+          records = array.read(record.flowset_data)
+
+          records.each do |r|
+            event = LogStash::Event.new(
+              "@timestamp" => Time.at(flowset.unix_sec).utc,
+              @target => {}
+            )
+
+            # Fewer fields in the v9 header
+            ['version', 'flow_seq_num'].each do |f|
+              event[@target][f] = flowset[f]
+            end
+
+            event[@target]['flowset_id'] = record.flowset_id
+
+            r.each_pair do |k,v|
+              case k.to_s
+              when /_switched$/
+                millis = flowset.uptime - v
+                seconds = flowset.unix_sec - (millis / 1000)
+                # v9 did away with the nanosecs field
+                micros = 1000000 - (millis % 1000)
+                event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
+              else
+                event[@target][k.to_s] = v
+              end
+            end
+
+            yield event
+          end
+        else
+          @logger.warn("Unsupported flowset id #{record.flowset_id}")
+        end
+      end
+    end
+  end # def filter
+
+  private
+  def uint_field(length, default)
+    # If length is 4, return :uint32, etc. and use default if length is 0
+    ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
+  end # def uint_field
+
+  private
+  def netflow_field_for(type, length)
+    case type
+    when 1
+      [[uint_field(length, 4), :in_bytes]]
+    when 2
+      [[uint_field(length, 4), :in_pkts]]
+    when 3
+      [[uint_field(length, 4), :flows]]
+    when 4
+      [[:uint8, :protocol]]
+    when 5
+      [[:uint8, :src_tos]]
+    when 6
+      [[:uint8, :tcp_flags]]
+    when 7
+      [[:uint16, :l4_src_port]]
+    when 8
+      [[:ip4_addr, :ipv4_src_addr]]
+    when 9
+      [[:uint8, :src_mask]]
+    when 10
+      [[uint_field(length, 2), :input_snmp]]
+    when 11
+      [[:uint16, :l4_dst_port]]
+    when 12
+      [[:ip4_addr, :ipv4_dst_addr]]
+    when 13
+      [[:uint8, :dst_mask]]
+    when 14
+      [[uint_field(length, 2), :output_snmp]]
+    when 15
+      [[:ip4_addr, :ipv4_next_hop]]
+    when 16
+      [[uint_field(length, 2), :src_as]]
+    when 17
+      [[uint_field(length, 2), :dst_as]]
+    when 18
+      [[:ip4_addr, :bgp_ipv4_next_hop]]
+    when 19
+      [[uint_field(length, 4), :mul_dst_pkts]]
+    when 20
+      [[uint_field(length, 4), :mul_dst_bytes]]
+    when 21
+      [[:uint32, :last_switched]]
+    when 22
+      [[:uint32, :first_switched]]
+    when 23
+      [[uint_field(length, 4), :out_bytes]]
+    when 24
+      [[uint_field(length, 4), :out_pkts]]
+    when 25
+      [[:uint16, :min_pkt_length]]
+    when 26
+      [[:uint16, :max_pkg_length]]
+    when 27
+      [[:ip6_addr, :ipv6_src_addr]]
+    when 28
+      [[:ip6_addr, :ipv6_dst_addr]]
+    when 29
+      [[:uint8, :ipv6_src_mask]]
+    when 30
+      [[:uint8, :ipv6_dst_mask]]
+    when 31
+      [[:uint24, :ipv6_flow_label]]
+    when 32
+      [[:uint16, :icmp_type]]
+    when 33
+      [[:uint8, :mul_igmp_type]]
+    when 34
+      [[:uint32, :sampling_interval]]
+    when 35
+      [[:uint8, :sampling_algorithm]]
+    when 36
+      [[:uint16, :flow_active_timeout]]
+    when 37
+      [[:uint16, :flow_inactive_timeout]]
+    when 38
+      [[:uint8, :engine_type]]
+    when 39
+      [[:uint8, :engine_id]]
+    when 40
+      [[uint_field(length, 4), :total_bytes_exp]]
+    when 41
+      [[uint_field(length, 4), :total_pkts_exp]]
+    when 42
+      [[uint_field(length, 4), :total_flows_exp]]
+    when 43 # Vendor specific field
+      [[:skip, nil, {:length => length}]]
+    when 44
+      [[:ip4_addr, :ipv4_src_prefix]]
+    when 45
+      [[:ip4_addr, :ipv4_dst_prefix]]
+    when 46
+      [[:uint8, :mpls_top_label_type]]
+    when 47
+      [[:uint32, :mpls_top_label_ip_addr]]
+    when 48
+      [[uint_field(length, 4), :flow_sampler_id]]
+    when 49
+      [[:uint8, :flow_sampler_mode]]
+    when 50
+      [[:uint32, :flow_sampler_random_interval]]
+    when 51 # Vendor specific field
+      [[:skip, nil, {:length => length}]]
+    when 52
+      [[:uint8, :min_ttl]]
+    when 53
+      [[:uint8, :max_ttl]]
+    when 54
+      [[:uint16, :ipv4_ident]]
+    when 55
+      [[:uint8, :dst_tos]]
+    when 56
+      [[:mac_addr, :in_src_mac]]
+    when 57
+      [[:mac_addr, :out_dst_mac]]
+    when 58
+      [[:uint16, :src_vlan]]
+    when 59
+      [[:uint16, :dst_vlan]]
+    when 60
+      [[:uint8, :ip_protocol_version]]
+    when 61
+      [[:uint8, :direction]]
+    when 62
+      [[:ip6_addr, :ipv6_next_hop]]
+    when 63
+      [[:ip6_addr, :bgp_ipv6_next_hop]]
+    when 64
+      [[:uint32, :ipv6_option_headers]]
+    when 65..69 # Vendor specific fields
+      [[:skip, nil, {:length => length}]]
+    when 80
+      [[:mac_addr, :in_dst_mac]]
+    when 81
+      [[:mac_addr, :out_src_mac]]
+    when 82
+      [[:string, :if_name, {:length => length, :trim_padding => true}]]
+    when 83
+      [[:string, :if_desc, {:length => length, :trim_padding => true}]]
+    else
+      @logger.warn("Unsupported field", :type => type, :length => length)
+      nil
+    end
+  end # def netflow_field_for
+end # class LogStash::Filters::Netflow