logstash-lite 0.2.20101118134500

data/etc/redhat/logstash.sysconfig

@@ -0,0 +1,18 @@
+# LogStash configuration
+
+# Run the indexer. This can be set to false to disable the indexer (and
+# searching) and only parse log lines.
+INDEXER=true
+
+# Run one parser. For maximum throughput, run one parser per core.
+# This can be set to 0 to disable the parser.
+PARSERS=1
+
+# Path to config file
+CONFIG=/opt/logstash/etc/logstashd.yaml
+
+# Set to true for very verbose debugging output
+DEBUG=false
+
+# Run as the logstash user
+USER=logstash

data/etc/tograylog.yaml

@@ -0,0 +1,37 @@
+--- 
+inputs:
+  linux-syslog:
+  - /var/log/messages
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access:
+  - /var/log/apache2/access.log
+  - /home/jls/logs/access_log
+  apache-error:
+  - /var/log/apache2/error.log
+  - /home/jls/logs/error_log
+filters:
+- grok:
+    linux-syslog: # for logs of type 'linux-syslog'
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs of type 'apache-error'
+      patterns:
+      - %{COMBINEDAPACHELOG}
+    nagios:
+      patterns:
+      - %{NAGIOSLOGLINE}
+- date:
+    linux-syslog:  # for logs of type 'linux-syslog'
+      # Look for a field 'timestamp' with this format, parse and it for the timestamp
+      # This field comes from the SYSLOGLINE pattern
+      timestamp: "%b %e %H:%M:%S"
+      timestamp8601: ISO8601
+    apache-access:
+      timestamp: "%d/%b/%Y:%H:%M:%S %Z"
+    nagios:
+      epochtime: %s
+outputs:
+- stdout:///
+- gelf://localhost/

data/examples/test.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+#
+# How to trigger the 'evil ip' message:
+# % logger -t "pantscon" "naughty host 14.33.24.55 $RANDOM"
+
+require "rubygems"
+require "logstash/agent"
+
+class MyAgent < LogStash::Agent
+  def receive(event)
+    filter(event)  # Invoke any filters
+
+    return unless event["progname"][0] == "pantscon"
+    return unless event.message =~ /naughty host/
+    event["IP"].each do |ip|
+      next unless ip.length > 0
+      puts "Evil IP: #{ip}"
+    end
+  end # def receive
+end # class MyAgent
+
+# Read a local file, parse it, and react accordingly (see MyAgent#receive)
+agent = MyAgent.new({
+  "input" => [
+    "/var/log/messages",
+  ],
+  "filter" => [ "grok" ],
+})
+agent.run
+
+# Read messages that we expect to be parsed by another agent. Reads
+# a particular AMQP topic for messages
+#agent = MyAgent.new({
+  #"input" => [
+    #"amqp://localhost/topic/parsed",
+  #]
+#})
+#agent.run

data/lib/logstash.rb

@@ -0,0 +1,3 @@
+require "logstash/namespace"
+require "logstash/agent"
+require "logstash/event"

data/lib/logstash/agent.rb

@@ -0,0 +1,116 @@
+require "eventmachine"
+require "eventmachine-tail"
+require "logstash/namespace"
+require "logstash/inputs"
+require "logstash/outputs"
+require "logstash/filters"
+require "logstash/logging"
+
+# Collect logs, ship them out.
+class LogStash::Agent
+  attr_reader :config
+
+  def initialize(config)
+    @logger = LogStash::Logger.new(STDERR)
+
+    @config = config
+    @outputs = []
+    @inputs = []
+    @filters = []
+    # Config should have:
+    # - list of logs to monitor
+    #   - log config
+    # - where to ship to
+  end # def initialize
+
+  # Register any event handlers with EventMachine
+  # Technically, this agent could listen for anything (files, sockets, amqp,
+  # stomp, etc).
+  protected
+  def register
+    # TODO(sissel): warn when no inputs and no outputs are defined.
+    # TODO(sissel): Refactor this madness into a config lib
+    
+    if (["inputs", "outputs"] & @config.keys).length == 0
+      $stderr.puts "No inputs or no outputs configured. This probably isn't what you want."
+    end
+
+    # Register input and output stuff
+    if @config.include?("inputs")
+      inputs = @config["inputs"]
+      inputs.each do |value|
+        # If 'url' is an array, then inputs is a hash and the key is the type
+        if inputs.is_a?(Hash)
+          type, urls = value
+        else
+          raise "config error, no type for url #{urls.inspect}"
+        end
+
+        # url could be a string or an array.
+        urls = [urls] if !urls.is_a?(Array)
+
+        urls.each do |url|
+          @logger.debug("Using input #{url} of type #{type}")
+          input = LogStash::Inputs.from_url(url, type) { |event| receive(event) }
+          input.register
+          @inputs << input
+        end
+      end # each input
+    end
+
+    if @config.include?("filters")
+      filters = @config["filters"]
+      filters.collect { |x| x.to_a[0] }.each do |filter|
+        name, value = filter
+        @logger.debug("Using filter #{name} => #{value.inspect}")
+        filter = LogStash::Filters.from_name(name, value)
+        filter.register
+        @filters << filter
+      end # each filter
+    end
+
+    if @config.include?("outputs")
+      @config["outputs"].each do |url|
+        @logger.debug("Using output #{url}")
+        output = LogStash::Outputs.from_url(url)
+        output.register
+        @outputs << output
+      end # each output
+    end
+  end # def register
+
+  public
+  def run
+    EventMachine.run do
+      self.register
+    end # EventMachine.run
+  end # def run
+
+  protected
+  def filter(event)
+    @filters.each do |f|
+      # TODO(sissel): Add ability for a filter to cancel/drop a message
+      f.filter(event)
+      if event.cancelled?
+        break
+      end
+    end
+  end # def filter
+
+  protected
+  def output(event)
+    @outputs.each do |o|
+      o.receive(event)
+    end # each output
+  end # def output
+
+  protected
+  # Process a message
+  def receive(event)
+    filter(event)
+
+    if !event.cancelled?
+      output(event)
+    end
+  end # def input
+end # class LogStash::Components::Agent

data/lib/logstash/event.rb

@@ -0,0 +1,70 @@
+require "json"
+require "logstash/time"
+require "uri"
+
+# General event type. Will expand this in the future.
+module LogStash; class Event
+  def initialize(data)
+    @cancelled = false
+    @data = {
+      "@source" => "unknown",
+      "@type" => nil,
+      "@tags" => [],
+      "@fields" => {},
+    }.merge(data)
+
+    if !@data.include?("@timestamp")
+      @data["@timestamp"] = LogStash::Time.now.utc.to_iso8601
+    end
+  end # def initialize
+
+  def self.from_json(json)
+    return Event.new(JSON.parse(json))
+  end # def self.from_json
+
+  def cancel
+    @cancelled = true
+  end
+
+  def cancelled?
+    return @cancelled
+  end
+
+  def to_s
+    return "#{timestamp} #{source}: #{message}"
+  end # def to_s
+
+  def timestamp; @data["@timestamp"]; end # def timestamp
+  def timestamp=(val); @data["@timestamp"] = val; end # def timestamp=
+
+  def source; @data["@source"]; end # def source
+  def source=(val) 
+    if val.is_a?(URI)
+      @data["@source"] = val.to_s
+      @data["@source_host"] = val.host
+      @data["@source_path"] = val.path
+    else
+      @data["@source"] = val
+    end
+  end # def source=
+
+  def message; @data["@message"]; end # def message
+  def message=(val); @data["@message"] = val; end # def message=
+
+  def type; @data["@type"]; end # def type
+  def type=(val); @data["@type"] = val; end # def type=
+
+  def tags; @data["@tags"]; end # def tags
+  def tags=(val); @data["@tags"] = val; end # def tags=
+
+  # field-related access
+  def [](key); @data["@fields"][key] end # def []
+  def []=(key, value); @data["@fields"][key] = value end # def []=
+  def fields; return @data["@fields"] end # def fields
+  
+  def to_json; return @data.to_json end # def to_json
+
+  def to_hash; return @data end # def to_hash
+
+  def include?(key); return @data.include?(key) end
+end; end # class LogStash::Event

data/lib/logstash/filters.rb

@@ -0,0 +1,17 @@
+
+require "logstash/namespace"
+
+module LogStash::Filters
+  def self.from_name(name, *args)
+    # TODO(sissel): Add error handling
+    # TODO(sissel): Allow plugin paths
+    klass = name.capitalize
+
+    # Load the class if we haven't already.
+    require "logstash/filters/#{name}"
+
+    # Get the class name from the Filters namespace and create a new instance.
+    # for name == 'foo' this will call LogStash::Filters::Foo.new
+    LogStash::Filters.const_get(klass).new(*args)
+  end # def from_url
+end # module LogStash::Filters

data/lib/logstash/filters/base.rb

@@ -0,0 +1,17 @@
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::Filters::Base
+  def initialize(config = {})
+    @logger = LogStash::Logger.new(STDERR)
+    @config = config
+  end # def initialize
+
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  def filter(event)
+    raise "#{self.class}#filter must be overidden"
+  end # def filter
+end # class LogStash::Filters::Base

data/lib/logstash/filters/date.rb

@@ -0,0 +1,59 @@
+require "logstash/filters/base"
+require "logstash/time"
+
+class LogStash::Filters::Date < LogStash::Filters::Base
+  # The 'date' filter will take a value from your event and use it as the
+  # event timestamp. This is useful for parsing logs generated on remote
+  # servers or for importing old logs.
+  #
+  # The config looks like this:
+  #
+  # filters:
+  #   date:
+  #     <type>:
+  #       <fieldname>: <format>
+  #     <type>
+  #       <fieldname>: <format>
+  #
+  # The format is whatever is supported by Ruby's DateTime.strptime
+  def initialize(config = {})
+    super
+
+    @types = Hash.new { |h,k| h[k] = [] }
+  end # def initialize
+
+  def register
+    @config.each do |type, typeconfig|
+      @logger.debug "Setting type #{type.inspect} to the config #{typeconfig.inspect}"
+      raise "date filter type \"#{type}\" defined more than once" unless @types[type].empty?
+      @types[type] = typeconfig
+    end # @config.each
+  end # def register
+
+  def filter(event)
+    @logger.debug "DATE FILTER: received event of type #{event.type}"
+    return unless @types.member?(event.type)
+    @types[event.type].each do |field, format|
+      @logger.debug "DATE FILTER: type #{event.type}, looking for field #{field.inspect} with format #{format.inspect}"
+      # TODO(sissel): check event.message, too.
+      if event.fields.member?(field)
+        fieldvalue = event.fields[field]
+        fieldvalue = [fieldvalue] if fieldvalue.is_a?(String)
+        fieldvalue.each do |value|
+          begin
+            case format
+              when "ISO8601"
+                time = DateTime.parse(value)
+              else
+                time = DateTime.strptime(value, format)
+            end
+            event.timestamp = LogStash::Time.to_iso8601(time)
+            @logger.debug "Parsed #{value.inspect} as #{event.timestamp}"
+          rescue
+            @logger.warn "Failed parsing date #{value.inspect} from field #{field} with format #{format.inspect}: #{$!}"
+          end
+        end # fieldvalue.each 
+      end # if this event has a field we expect to be a timestamp
+    end # @types[event.type].each
+  end # def filter
+end # class LogStash::Filters::Date

data/lib/logstash/filters/field.rb

@@ -0,0 +1,29 @@
+require "logstash/filters/base"
+require "ostruct"
+
+class LogStash::Filters::Field < LogStash::Filters::Base
+  class EvalSpace < OpenStruct
+    def get_binding
+      return binding
+    end
+  end
+
+  def initialize(config = {})
+    super
+  end # def initialize
+
+  def register
+    # nothing to do
+  end # def register
+
+  def filter(event)
+    data = EvalSpace.new(event.to_hash)
+
+    @config.each do |condition|
+      if data.instance_eval(condition)
+        return # This event is OK, matches the condition.
+      end
+    end
+    event.cancel
+  end
+end # class LogStash::Filters::Field

data/lib/logstash/filters/grok.rb

@@ -0,0 +1,74 @@
+require "logstash/filters/base"
+
+gem "jls-grok", ">=0.2.3071"
+require "grok" # rubygem 'jls-grok'
+
+class LogStash::Filters::Grok < LogStash::Filters::Base
+  def initialize(config = {})
+    super
+
+    @grokpiles = {}
+  end # def initialize
+
+  def register
+    # TODO(sissel): Make patterns files come from the config
+    @config.each do |type, typeconfig|
+      @logger.debug("Registering type with grok: #{type}")
+      pile = Grok::Pile.new
+      patterndir = "#{File.dirname(__FILE__)}/../../../patterns/*"
+      Dir.glob(patterndir).each do |path|
+        pile.add_patterns_from_file(path)
+      end
+      typeconfig["patterns"].each do |pattern|
+        groks = pile.compile(pattern)
+        @logger.debug(["Compiled pattern", pattern, groks[-1].expanded_pattern])
+      end
+      @grokpiles[type] = pile
+    end # @config.each
+  end # def register
+
+  def filter(event)
+    # parse it with grok
+    message = event.message
+    match = false
+
+    if event.type
+      if @grokpiles.include?(event.type)
+        @logger.debug(["Running grok filter", event])
+        pile = @grokpiles[event.type]
+        grok, match = pile.match(message)
+      end # @grokpiles.include?(event.type)
+      # TODO(2.0): support grok pattern discovery
+    else
+      @logger.info("Unknown type for #{event.source} (type: #{event.type})")
+      @logger.debug(event.to_hash)
+    end
+
+    if match
+      match.each_capture do |key, value|
+        if key.include?(":")
+          key = key.split(":")[1]
+        end
+        if event.message == value
+          # Skip patterns that match the entire line
+          @logger.debug("Skipping capture '#{key}' since it matches the whole line.")
+          next
+        end
+
+        if event.fields[key].is_a?(String)
+          event.fields[key] = [event.fields[key]]
+        elsif event.fields[key] == nil
+          event.fields[key] = []
+        end
+
+        event.fields[key] << value
+      end
+    else
+      # Tag this event if we can't parse it. We can use this later to
+      # reparse+reindex logs if we improve the patterns given .
+      event.tags << "_grokparsefailure"
+    end
+
+    @logger.debug(["Event now: ", event.to_hash])
+  end # def filter
+end # class LogStash::Filters::Grok