logstash-lib 1.3.2

data/lib/logstash/codecs/netflow/util.rb

@@ -0,0 +1,212 @@
+# encoding: utf-8
+require "bindata"
+require "ipaddr"
+
+class IP4Addr < BinData::Primitive
+  endian :big
+  uint32 :storage
+
+  def set(val)
+    ip = IPAddr.new(val)
+    if ! ip.ipv4?
+      raise ArgumentError, "invalid IPv4 address '#{val}'"
+    end
+    self.storage = ip.to_i
+  end
+
+  def get
+    IPAddr.new_ntoh([self.storage].pack('N')).to_s
+  end
+end
+
+class IP6Addr < BinData::Primitive
+  endian  :big
+  uint128 :storage
+
+  def set(val)
+    ip = IPAddr.new(val)
+    if ! ip.ipv6?
+      raise ArgumentError, "invalid IPv6 address `#{val}'"
+    end
+    self.storage = ip.to_i
+  end
+
+  def get
+    IPAddr.new_ntoh((0..7).map { |i|
+      (self.storage >> (112 - 16 * i)) & 0xffff
+    }.pack('n8')).to_s
+  end
+end
+
+class MacAddr < BinData::Primitive
+  array :bytes, :type => :uint8, :initial_length => 6
+
+  def set(val)
+    ints = val.split(/:/).collect { |int| int.to_i(16) }
+    self.bytes = ints
+  end
+
+  def get
+    self.bytes.collect { |byte| byte.to_s(16) }.join(":")
+  end
+end
+
+class Header < BinData::Record
+  endian :big
+  uint16 :version
+end
+
+class Netflow5PDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :flow_records
+  uint32 :uptime
+  uint32 :unix_sec
+  uint32 :unix_nsec
+  uint32 :flow_seq_num
+  uint8  :engine_type
+  uint8  :engine_id
+  bit2   :sampling_algorithm
+  bit14  :sampling_interval
+  array  :records, :initial_length => :flow_records do
+    ip4_addr :ipv4_src_addr
+    ip4_addr :ipv4_dst_addr
+    ip4_addr :ipv4_next_hop
+    uint16   :input_snmp
+    uint16   :output_snmp
+    uint32   :in_pkts
+    uint32   :in_bytes
+    uint32   :first_switched
+    uint32   :last_switched
+    uint16   :l4_src_port
+    uint16   :l4_dst_port
+    skip     :length => 1
+    uint8    :tcp_flags # Split up the TCP flags maybe?
+    uint8    :protocol
+    uint8    :src_tos
+    uint16   :src_as
+    uint16   :dst_as
+    uint8    :src_mask
+    uint8    :dst_mask
+    skip     :length => 2
+  end
+end
+
+class TemplateFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
+    uint16 :template_id
+    uint16 :field_count
+    array  :fields, :initial_length => :field_count do
+      uint16 :field_type
+      uint16 :field_length
+    end
+  end
+end
+
+class OptionFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :scope_length
+    uint16 :option_length
+    array  :scope_fields, :initial_length => lambda { scope_length / 4 } do
+      uint16 :field_type
+      uint16 :field_length
+    end
+    array  :option_fields, :initial_length => lambda { option_length / 4 } do
+      uint16 :field_type
+      uint16 :field_length
+    end
+  end
+  skip   :length => lambda { templates.length.odd? ? 2 : 0 }
+end
+
+class Netflow9PDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :flow_records
+  uint32 :uptime
+  uint32 :unix_sec
+  uint32 :flow_seq_num
+  uint32 :source_id
+  array  :records, :read_until => :eof do
+    uint16 :flowset_id
+    uint16 :flowset_length
+    choice :flowset_data, :selection => :flowset_id do
+      template_flowset 0
+      option_flowset   1
+      string           :default, :read_length => lambda { flowset_length - 4 }
+    end
+  end
+end
+
+# https://gist.github.com/joshaven/184837
+class Vash < Hash
+  def initialize(constructor = {})
+    @register ||= {}
+    if constructor.is_a?(Hash)
+      super()
+      merge(constructor)
+    else
+      super(constructor)
+    end
+  end
+
+  alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+  alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
+
+  def [](key)
+    sterilize(key)
+    clear(key) if expired?(key)
+    regular_reader(key)
+  end
+
+  def []=(key, *args)
+    if args.length == 2
+      value, ttl = args[1], args[0]
+    elsif args.length == 1
+      value, ttl = args[0], 60
+    else
+      raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
+                           "Example Usage:  volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
+    end
+    sterilize(key)
+    ttl(key, ttl)
+    regular_writer(key, value)
+  end
+
+  def merge(hsh)
+    hsh.map {|key,value| self[sterile(key)] = hsh[key]}
+    self
+  end
+
+  def cleanup!
+    now = Time.now.to_i
+    @register.map {|k,v| clear(k) if v < now}
+  end
+
+  def clear(key)
+    sterilize(key)
+    @register.delete key
+    self.delete key
+  end
+
+  private
+  def expired?(key)
+    Time.now.to_i > @register[key].to_i
+  end
+
+  def ttl(key, secs=60)
+    @register[key] = Time.now.to_i + secs.to_i
+  end
+
+  def sterile(key)
+    String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
+  end
+
+  def sterilize(key)
+    key = sterile(key)
+  end
+end
+

data/lib/logstash/codecs/noop.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::Noop < LogStash::Codecs::Base
+  config_name "noop"
+
+  milestone 1
+  
+  public
+  def decode(data)
+    yield data
+  end # def decode
+
+  public
+  def encode(data)
+    @on_event.call data
+  end # def encode
+
+end # class LogStash::Codecs::Noop

data/lib/logstash/codecs/oldlogstashjson.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::OldLogStashJSON < LogStash::Codecs::Base
+  config_name "oldlogstashjson"
+  milestone 2
+
+  # Map from v0 name to v1 name.
+  # Note: @source is gone and has no similar field.
+  V0_TO_V1 = {"@timestamp" => "@timestamp", "@message" => "message",
+              "@tags" => "tags", "@type" => "type",
+              "@source_host" => "host", "@source_path" => "path"}
+
+  public
+  def decode(data)
+    begin
+      obj = JSON.parse(data.force_encoding("UTF-8"))
+    rescue JSON::ParserError => e
+      @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
+      yield LogStash::Event.new("message" => data)
+      return
+    end
+
+    h  = {}
+
+    # Convert the old logstash schema to the new one.
+    V0_TO_V1.each do |key, val|
+      h[val] = obj[key] if obj.include?(key)
+    end
+
+    h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
+    yield LogStash::Event.new(h)
+  end # def decode
+
+  public
+  def encode(data)
+    h  = {}
+
+    # Convert the new logstash schema to the old one.
+    V0_TO_V1.each do |key, val|
+      h[key] = data[val] if data.include?(val)
+    end
+
+    data.to_hash.each do |field, val|
+      # TODO: might be better to V1_TO_V0 = V0_TO_V1.invert during
+      # initialization than V0_TO_V1.has_value? within loop
+      next if field == "@version" or V0_TO_V1.has_value?(field)
+      h["@fields"] = {} if h["@fields"].nil?
+      h["@fields"][field] = val
+    end
+
+    # Tack on a \n because JSON outputs 1.1.x had them.
+    @on_event.call(h.to_json + "\n")
+  end # def encode
+
+end # class LogStash::Codecs::OldLogStashJSON

data/lib/logstash/codecs/plain.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/util/charset"
+
+# The "plain" codec is for plain text with no delimiting between events.
+#
+# This is mainly useful on inputs and outputs that already have a defined
+# framing in their transport protocol (such as zeromq, rabbitmq, redis, etc)
+class LogStash::Codecs::Plain < LogStash::Codecs::Base
+  config_name "plain"
+  milestone 3
+
+  # Set the message you which to emit for each event. This supports sprintf
+  # strings.
+  #
+  # This setting only affects outputs (encoding of events).
+  config :format, :validate => :string
+
+  # The character encoding used in this input. Examples include "UTF-8"
+  # and "cp1252"
+  #
+  # This setting is useful if your log files are in Latin-1 (aka cp1252)
+  # or in another character set other than UTF-8.
+  #
+  # This only affects "plain" format logs since json is UTF-8 already.
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def register
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+
+  public
+  def decode(data)
+    yield LogStash::Event.new("message" => @converter.convert(data))
+  end # def decode
+
+  public
+  def encode(data)
+    if data.is_a? LogStash::Event and @format
+      @on_event.call(data.sprintf(@format))
+    else
+      @on_event.call(data.to_s)
+    end
+  end # def encode
+
+end # class LogStash::Codecs::Plain

data/lib/logstash/codecs/rubydebug.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::RubyDebug < LogStash::Codecs::Base
+  config_name "rubydebug"
+  milestone 3
+
+  def register
+    require "ap"
+  end
+
+  public
+  def decode(data)
+    raise "Not implemented"
+  end # def decode
+
+  public
+  def encode(data)
+    @on_event.call(data.to_hash.awesome_inspect + "\n")
+  end # def encode
+
+end # class LogStash::Codecs::Dots

data/lib/logstash/codecs/spool.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::Spool < LogStash::Codecs::Base
+  config_name 'spool'
+  milestone 1
+  config :spool_size, :validate => :number, :default => 50
+
+  attr_reader :buffer
+
+  public
+  def decode(data)
+    data.each do |event|
+      yield event
+    end
+  end # def decode
+
+  public
+  def encode(data)
+    @buffer = [] if @buffer.nil?
+    #buffer size is hard coded for now until a 
+    #better way to pass args into codecs is implemented
+    if @buffer.length >= @spool_size
+      @on_event.call @buffer
+      @buffer = []
+    else
+      @buffer << data
+    end
+  end # def encode
+
+  public
+  def teardown
+    if !@buffer.nil? and @buffer.length > 0
+      @on_event.call @buffer
+    end
+    @buffer = []
+  end
+end # class LogStash::Codecs::Spool

data/lib/logstash/config/Makefile

@@ -0,0 +1,4 @@
+
+#ragel -R grammar.rl
+grammar.rb: grammar.treetop
+	tt grammar.treetop

data/lib/logstash/config/config_ast.rb

@@ -0,0 +1,380 @@
+# encoding: utf-8
+require "treetop"
+class Treetop::Runtime::SyntaxNode
+  def compile
+    return "" if elements.nil?
+    return elements.collect(&:compile).reject(&:empty?).join("")
+  end
+
+  def recursive_inject(results=[], &block)
+    if !elements.nil?
+      elements.each do |element|
+        if block.call(element)
+          results << element
+        else
+          element.recursive_inject(results, &block)
+        end
+      end
+    end
+    return results
+  end
+
+  def recursive_select(klass)
+    return recursive_inject { |e| e.is_a?(klass) }
+  end
+
+  def recursive_inject_parent(results=[], &block)
+    if !parent.nil?
+      if block.call(parent)
+        results << parent
+      else
+        parent.recursive_inject_parent(results, &block)
+      end
+    end
+    return results
+  end
+
+  def recursive_select_parent(results=[], klass)
+    return recursive_inject_parent(results) { |e| e.is_a?(klass) }
+  end
+end
+
+module LogStash; module Config; module AST 
+  class Node < Treetop::Runtime::SyntaxNode; end
+  class Config < Node
+    def compile
+      # TODO(sissel): Move this into config/config_ast.rb
+      code = []
+      code << "@inputs = []"
+      code << "@filters = []"
+      code << "@outputs = []"
+      sections = recursive_select(LogStash::Config::AST::PluginSection)
+      sections.each do |s|
+        code << s.compile_initializer
+      end
+
+      # start inputs
+      #code << "class << self"
+      definitions = []
+        
+      ["filter", "output"].each do |type|
+        #definitions << "def #{type}(event)"
+        definitions << "@#{type}_func = lambda do |event, &block|"
+        if type == "filter"
+          definitions << "  extra_events = []"
+        end
+
+        definitions << "  @logger.info? && @logger.info(\"#{type} received\", :event => event)"
+        sections.select { |s| s.plugin_type.text_value == type }.each do |s|
+          definitions << s.compile.split("\n", -1).map { |e| "  #{e}" }
+        end
+
+        if type == "filter"
+          definitions << "  extra_events.each(&block)"
+        end
+        definitions << "end"
+      end
+
+      code += definitions.join("\n").split("\n", -1).collect { |l| "  #{l}" }
+      #code << "end"
+      return code.join("\n")
+    end
+  end
+
+  class Comment < Node; end
+  class Whitespace < Node; end
+  class PluginSection < Node
+    @@i = 0
+    # Generate ruby code to initialize all the plugins.
+    def compile_initializer
+      generate_variables
+      code = []
+      @variables.collect do |plugin, name|
+        code << "#{name} = #{plugin.compile_initializer}"
+        code << "@#{plugin.plugin_type}s << #{name}"
+      end
+      return code.join("\n")
+    end
+
+    def variable(object)
+      generate_variables
+      return @variables[object]
+    end
+
+    def generate_variables
+      return if !@variables.nil?
+      @variables = {}
+      plugins = recursive_select(Plugin)
+
+      plugins.each do |plugin|
+        # Unique number for every plugin.
+        @@i += 1
+        # store things as ivars, like @filter_grok_3
+        var = "@#{plugin.plugin_type}_#{plugin.plugin_name}_#{@@i}"
+        @variables[plugin] = var
+      end
+      return @variables
+    end
+
+  end
+
+  class Plugins < Node; end
+  class Plugin < Node
+    def plugin_type
+      if recursive_select_parent(Plugin).any?
+        return "codec"
+      else
+        return recursive_select_parent(PluginSection).first.plugin_type.text_value
+      end
+    end
+
+    def plugin_name
+      return name.text_value
+    end
+
+    def variable_name
+      return recursive_select_parent(PluginSection).first.variable(self)
+    end
+
+    def compile_initializer
+      # If any parent is a Plugin, this must be a codec.
+
+      if attributes.elements.nil?
+        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect})" << (plugin_type == "codec" ? "" : "\n")
+      else
+        settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
+
+        attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
+        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{attributes_code})" << (plugin_type == "codec" ? "" : "\n")
+      end
+    end
+
+    def compile
+      case plugin_type
+        when "input"
+          return "start_input(#{variable_name})"
+        when "filter"
+          # This is some pretty stupid code, honestly.
+          # I'd prefer much if it were put into the Pipeline itself
+          # and this should simply compile to 
+          #   #{variable_name}.filter(event)
+          return [
+            "newevents = []",
+            "extra_events.each do |event|",
+            "  #{variable_name}.filter(event) do |newevent|",
+            "    newevents << newevent",
+            "  end",
+            "end",
+            "extra_events += newevents",
+
+            "#{variable_name}.filter(event) do |newevent|",
+            "  extra_events << newevent",
+            "end",
+            "if event.cancelled?",
+            "  extra_events.each(&block)",
+            "  return",
+            "end",
+          ].map { |l| "#{l}\n" }.join("")
+        when "output"
+          return "#{variable_name}.handle(event)\n"
+        when "codec"
+          settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
+          attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
+          return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{attributes_code})"
+      end
+    end
+  end
+
+  class Name < Node
+    def compile
+      return text_value.inspect
+    end
+  end
+  class Attribute < Node
+    def compile
+      return %Q(#{name.compile} => #{value.compile})
+    end
+  end
+  class RValue < Node; end
+  class Value < RValue; end
+
+  module Unicode
+    def self.wrap(text)
+      return "(" + text.inspect + ".force_encoding(\"UTF-8\")" + ")"
+    end
+  end
+
+  class Bareword < Value
+    def compile
+      return Unicode.wrap(text_value)
+    end
+  end
+  class String < Value
+    def compile
+      return Unicode.wrap(text_value[1...-1])
+    end
+  end
+  class RegExp < Value
+    def compile
+      return "Regexp.new(" + Unicode.wrap(text_value[1...-1]) + ")"
+    end
+  end
+  class Number < Value
+    def compile
+      return text_value
+    end
+  end
+  class Array < Value
+    def compile
+      return "[" << recursive_select(Value).collect(&:compile).reject(&:empty?).join(", ") << "]"
+    end
+  end
+  class Hash < Value
+    def compile
+      return "{" << recursive_select(HashEntry).collect(&:compile).reject(&:empty?).join(", ") << "}"
+    end
+  end
+  class HashEntries < Node; end
+  class HashEntry < Node
+    def compile
+      return %Q(#{name.compile} => #{value.compile})
+    end
+  end
+
+  class BranchOrPlugin < Node; end
+
+  class Branch < Node
+    def compile
+      return super + "end\n"
+    end
+  end
+  class If < Node
+    def compile
+      children = recursive_inject { |e| e.is_a?(Branch) || e.is_a?(Plugin) }
+      return "if #{condition.compile}\n" \
+        << children.collect(&:compile).map { |s| s.split("\n", -1).map { |l| "  " + l }.join("\n") }.join("") << "\n"
+    end
+  end
+  class Elsif < Node
+    def compile
+      children = recursive_inject { |e| e.is_a?(Branch) || e.is_a?(Plugin) }
+      return "elsif #{condition.compile}\n" \
+        << children.collect(&:compile).map { |s| s.split("\n", -1).map { |l| "  " + l }.join("\n") }.join("") << "\n"
+    end
+  end
+  class Else < Node
+    def compile
+      children = recursive_inject { |e| e.is_a?(Branch) || e.is_a?(Plugin) }
+      return "else\n" \
+        << children.collect(&:compile).map { |s| s.split("\n", -1).map { |l| "  " + l }.join("\n") }.join("") << "\n"
+    end
+  end
+
+  class Condition < Node
+    def compile
+      return "(#{super})"
+    end
+  end
+
+  module Expression
+    def compile
+      return "(#{super})"
+    end
+  end
+
+  module NegativeExpression
+    def compile
+      return "!(#{super})"
+    end
+  end
+
+  module ComparisonExpression; end
+
+  module InExpression
+    def compile
+      item, list = recursive_select(LogStash::Config::AST::RValue)
+      return "(x = #{list.compile}; x.respond_to?(:include?) && x.include?(#{item.compile}))"
+    end
+  end
+
+  module NotInExpression
+    def compile
+      item, list = recursive_select(LogStash::Config::AST::RValue)
+      return "(x = #{list.compile}; !x.respond_to?(:include?) || !x.include?(#{item.compile}))"
+    end
+  end
+
+  class MethodCall < Node
+    def compile
+      arguments = recursive_inject { |e| [String, Number, Selector, Array, MethodCall].any? { |c| e.is_a?(c) } }
+      return "#{method.text_value}(" << arguments.collect(&:compile).join(", ") << ")"
+    end
+  end
+
+  class RegexpExpression < Node
+    def compile
+      operator = recursive_select(LogStash::Config::AST::RegExpOperator).first.text_value
+      item, regexp = recursive_select(LogStash::Config::AST::RValue)
+      # Compile strings to regexp's
+      if regexp.is_a?(LogStash::Config::AST::String)
+        regexp = "/#{regexp.text_value[1..-2]}/"
+      else
+        regexp = regexp.compile
+      end
+      return "(#{item.compile} #{operator} #{regexp})"
+    end
+  end
+
+  module ComparisonOperator 
+    def compile
+      return " #{text_value} "
+    end
+  end
+  module RegExpOperator
+    def compile
+      return " #{text_value} "
+    end
+  end
+  module BooleanOperator
+    def compile
+      return " #{text_value} "
+    end
+  end
+  class Selector < RValue
+    def compile
+      return "event[#{text_value.inspect}]"
+    end
+  end
+  class SelectorElement < Node; end
+end; end; end
+
+
+# Monkeypatch Treetop::Runtime::SyntaxNode's inspect method to skip
+# any Whitespace or SyntaxNodes with no children.
+class Treetop::Runtime::SyntaxNode
+  def _inspect(indent="")
+    em = extension_modules
+    interesting_methods = methods-[em.last ? em.last.methods : nil]-self.class.instance_methods
+    im = interesting_methods.size > 0 ? " (#{interesting_methods.join(",")})" : ""
+    tv = text_value
+    tv = "...#{tv[-20..-1]}" if tv.size > 20
+
+    indent +
+    self.class.to_s.sub(/.*:/,'') +
+      em.map{|m| "+"+m.to_s.sub(/.*:/,'')}*"" +
+      " offset=#{interval.first}" +
+      ", #{tv.inspect}" +
+      im +
+      (elements && elements.size > 0 ?
+        ":" +
+          (elements.select { |e| !e.is_a?(LogStash::Config::AST::Whitespace) && e.elements && e.elements.size > 0 }||[]).map{|e|
+      begin
+        "\n"+e.inspect(indent+"  ")
+      rescue  # Defend against inspect not taking a parameter
+        "\n"+indent+" "+e.inspect
+      end
+          }.join("") :
+        ""
+      )
+  end
+end