logstash-lite 0.2.20101119183130 → 0.2.20101120021802

data/etc/prod.yaml

@@ -11,6 +11,8 @@ inputs:
   - /var/log/apache2/access.log
   apache-error:
   - /var/log/apache2/error.log
+  testing:
+  - /tmp/logstashtest.log
 filters:
 - grok:
     linux-syslog: # for logs of type 'linux-syslog'
@@ -22,6 +24,12 @@ filters:
     nagios:
       patterns:
       - %{NAGIOSLOGLINE}
+    loggly:
+      patterns:
+      - %{JAVASTACKTRACEPART}
+    testing:
+      patterns:
+      - %{JAVASTACKTRACEPART}
 - date:
     linux-syslog:  # for logs of type 'linux-syslog'
       # Look for a field 'timestamp' with this format, parse and it for the timestamp
@@ -32,6 +40,13 @@ filters:
       timestamp: "%d/%b/%Y:%H:%M:%S %Z"
     nagios:
       epochtime: %s
+- multiline:
+    supervisorlogs:
+      pattern: ^\s
+      what: previous
+    testing:
+      pattern: ^\s
+      what: previous
 outputs:
 - stdout:///
 #- elasticsearch://localhost:9200/logstash/all

data/lib/logstash/event.rb

@@ -66,5 +66,24 @@ module LogStash; class Event
 
   def to_hash; return @data end # def to_hash
 
+  def overwrite(event)
+    @data = event.to_hash
+  end
+
   def include?(key); return @data.include?(key) end
+
+  # Append an event to this one.
+  def append(event)
+    self.message += "\n" + event.message 
+    self.tags |= event.tags
+
+    # Append all fields
+    event.fields.each do |name, value|
+      if event.fields.include?(name)
+        event.fields[name] |= value
+      else
+        event.fields[name] = value
+      end
+    end # event.fields.each
+  end
 end; end # class LogStash::Event

data/lib/logstash/filters/multiline.rb

@@ -0,0 +1,158 @@
+# multiline filter
+#
+# This filter will collapse multiline messages into a single event.
+# 
+
+require "logstash/filters/base"
+
+class LogStash::Filters::Multiline < LogStash::Filters::Base
+  # The 'date' filter will take a value from your event and use it as the
+  # event timestamp. This is useful for parsing logs generated on remote
+  # servers or for importing old logs.
+  #
+  # The config looks like this:
+  #
+  # filters:
+  # - multiline:
+  #     <type>:
+  #       pattern: <regexp>
+  #       what: next
+  #     <type>
+  #        pattern: <regexp>
+  #        what: previous
+  # 
+  # The 'regexp' should match what you believe to be an indicator that
+  # the field is part of a multi-line event
+  #
+  # The 'what' must be "previous" or "next" and indicates the relation
+  # to the multi-line event.
+  #
+  # For example, java stack traces are multiline and usually have the message
+  # starting at the far-left, then each subsequent line indented. Do this:
+  # 
+  # filters:
+  # - multiline:
+  #     somefiletype:
+  #       pattern: /^\s/
+  #       what: previous
+  #
+  # This says that any line starting with whitespace belongs to the previous line.
+  #
+  # Another example is C line continuations (backslash). Here's how to do that:
+  #
+  # filters:
+  # - multiline:
+  #     somefiletype:
+  #       pattern: /\\$/
+  #       what: next
+  #
+  def initialize(config = {})
+    super
+
+    @types = Hash.new { |h,k| h[k] = [] }
+    @pending = Hash.new
+  end # def initialize
+
+  def register
+    @config.each do |type, typeconfig|
+      # typeconfig will be a hash containing 'pattern' and 'what'
+      @logger.debug "Setting type #{type.inspect} to the config #{typeconfig.inspect}"
+      raise "type \"#{type}\" defined more than once" unless @types[type].empty?
+      @types[type] = typeconfig
+
+      if !typeconfig.include?("pattern")
+        @logger.fatal(["'multiline' filter config for type #{type} is missing" \
+                       " 'pattern' setting", typeconfig])
+      end
+
+      if !typeconfig.include?("what")
+        @logger.fatal(["'multiline' filter config for type #{type} is missing" \
+                       " 'what' setting", typeconfig])
+      end
+
+      if !["next", "previous"].include?(typeconfig["what"])
+        @logger.fatal(["'multiline' filter config for type #{type} has " \
+                       "invalid 'what' value. Must be 'next' or 'previous'",
+                       typeconfig])
+      end
+
+      begin
+        typeconfig["pattern"] = Regexp.new(typeconfig["pattern"])
+      rescue RegexpError => e
+        @logger.fatal(["Invalid pattern for multiline filter on type '#{type}'",
+                      typeconfig, e])
+      end
+
+    end # @config.each
+  end # def register
+
+  def filter(event)
+    return unless @types.member?(event.type)
+    typeconfig = @types[event.type]
+    match = typeconfig["pattern"].match(event.message)
+    key = [event.source, event.type]
+    pending = @pending[key]
+
+    @logger.info(["Reg: ", typeconfig["pattern"], event.message, match])
+    case typeconfig["what"]
+    when "previous"
+      if match
+        event.tags |= ["multiline"]
+        # previous previous line is part of this event.
+        # append it to the event and cancel it
+        if pending
+          pending.append(event)
+        else
+          @pending[key] = event
+        end
+        event.cancel
+      else
+        # this line is not part of the previous event
+        # if we have a pending event, it's done, send it.
+        # put the current event into pending
+        if pending
+          tmp = event.to_hash
+          event.overwrite(pending)
+          @pending[key] = LogStash::Event.new(tmp)
+        else
+          @pending[key] = event
+          event.cancel
+        end # if/else pending
+      end # if/else match
+    when "next"
+      if match
+        event.tags |= ["multiline"]
+        # this line is part of a multiline event, the next
+        # line will be part, too, put it into pending.
+        if pending
+          pending.append(event)
+        else
+          @pending[key] = event
+        end
+        event.cancel
+      else
+        # if we have something in pending, join it with this message
+        # and send it. otherwise, this is a new message and not part of
+        # multiline, send it.
+        if pending
+          pending.append(event)
+          event.overwrite(pending.to_hash)
+          @pending.delete(key)
+        end
+      end # if/else match
+    else
+      @logger.warn(["Unknown multiline 'what' value.", typeconfig])
+    end # case typeconfig["what"]
+    #end # @types[event.type].each
+  end # def filter
+
+  # flush any pending messages
+  def flush(source, type)
+    key = [source, type]
+    if @pending[key]
+      event = @pending[key]
+      @pending.delete(key)
+    end
+    return event
+  end
+end # class LogStash::Filters::Date

data/lib/logstash/inputs/file.rb

@@ -12,6 +12,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   end
 
   def register
+    @logger.info("Registering #{@url}")
     EventMachine::FileGlobWatchTail.new(@url.path, Reader, interval=60,
                                         exclude=[], receiver=self)
   end # def register

data/lib/logstash/web/lib/elasticsearch.rb

@@ -46,8 +46,9 @@ class LogStash::Web::ElasticSearch
       data["duration"] = Time.now - start_time
 
       # TODO(sissel): Plugin-ify this (Search filters!)
-      require "digest/md5"
-      data["hits"]["hits"].each do |hit|
+      #require "digest/md5"
+      #data["hits"]["hits"].each do |hit|
+      [].each do |hit|
         event = LogStash::Event.new(hit["_source"])
         event.to_hash.each do |key, value|
           next unless value.is_a?(String)

data/lib/logstash/web/public/js/logstash.js

@@ -88,7 +88,7 @@
         var a = params[p].split("=");
         var key = a[0]
         var value = a[1]
-        logstash.params[key] = value
+        logstash.params[key] = unescape(value)
       }
       logstash.search(logstash.params.q)
       return false;

data/lib/logstash/web/views/search/ajax.haml

@@ -35,4 +35,5 @@
       - @hits.reverse.each do |hit|
         %tr.event
           %td.timestamp&= hit["_source"]["@timestamp"]
-          %td.message{ :"data-full" => hit.to_json }&= hit["_source"]["@message"]
+          %td.message{ :"data-full" => hit.to_json }
+            %pre&= hit["_source"]["@message"]

data/lib/logstash/web/views/style.sass

@@ -22,13 +22,18 @@ body
   margin-top: 1em
 #content table.results
   font-family: monospace
-#content td.event
+#content td.message
+  vertical-align: top
+  padding: 1px
   padding-bottom: 3px
-  white-space: pre-wrap
+  pre
+    white-space: pre-wrap
+    margin: 0
 #content td.timestamp
   white-space: nowrap
-  font-size: 85%
-  text-align: top
+  padding: 1px
+  //font-size: 85%
+  vertical-align: top
 #content tr.selected
   background-color: #FCE69D !important
 #content tr.event:nth-child(2n)

data/patterns/java

@@ -0,0 +1,3 @@
+JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9]+
+JAVAFILE (?:[A-Za-z0-9_.-]+)
+JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-lite
 version: !ruby/object:Gem::Version 
-  hash: 40202238366243
+  hash: 40202240043587
   prerelease: false
   segments: 
   - 0
   - 2
-  - 20101119183130
-  version: 0.2.20101119183130
+  - 20101120021802
+  version: 0.2.20101120021802
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-19 00:00:00 -08:00
+date: 2010-11-20 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -76,6 +76,7 @@ files:
 - lib/logstash/filters.rb
 - lib/logstash/outputs.rb
 - lib/logstash/filters/grokdiscovery.rb
+- lib/logstash/filters/multiline.rb
 - lib/logstash/filters/grok.rb
 - lib/logstash/filters/base.rb
 - lib/logstash/filters/field.rb
@@ -183,6 +184,7 @@ files:
 - etc/logstash-shipper.yaml
 - patterns/linux-syslog
 - patterns/haproxy
+- patterns/java
 - patterns/grok-patterns
 - patterns/ruby
 - patterns/firewalls