logstash-lite 0.2.20110122143801 → 0.2.20110203130400

data/lib/logstash/event.rb

@@ -70,7 +70,20 @@ class LogStash::Event
 
   # field-related access
   public
-  def [](key); @data["@fields"][key] end # def []
+  def [](key)
+    # If the key isn't in fields and it starts with an "@" sign, get it out of data instead of fields
+    if ! @data["@fields"].has_key?(key) and key.slice(0,1) == "@"
+      return @data[key]
+    # Exists in @fields (returns value) or doesn't start with "@" (return null)
+    else
+      return @data["@fields"][key]
+    end
+  end # def []
+  
+  # TODO(sissel): the semantics of [] and []= are now different in that
+  # []= only allows you to assign to only fields (not metadata), but
+  # [] allows you to read fields and metadata.
+  # We should fix this. Metadata is really a namespace issue, anyway.
   def []=(key, value); @data["@fields"][key] = value end # def []=
   def fields; return @data["@fields"] end # def fields
   
@@ -103,4 +116,29 @@ class LogStash::Event
       end
     end # event.fields.each
   end # def append
+
+  # sprintf. This could use a better method name.
+  # The idea is to take an event and convert it to a string based on 
+  # any format values, delimited by ${foo} where 'foo' is a field or
+  # metadata member.
+  #
+  # For example, if the event has @type == "foo" and @source == "bar"
+  # then this string:
+  #   "type is ${@type} and source is #{@source}"
+  # will return
+  #   "type is foo and source is bar"
+  #
+  # If a ${name} value does not exist, then no substitution occurs.
+  #
+  # TODO(sissel): It is not clear what the value of a field that 
+  # is an array (or hash?) should be. Join by comma? Something else?
+  public
+  def sprintf(format)
+    result = format.gsub(/\$\{[@A-Za-z0-9_-]+\}/) do |match|
+      name = match[2..-2] # trim '${' and '}'
+      value = (self[name] or match)
+    end
+    #$stderr.puts "sprintf(#{format.inspect}) => #{result.inspect}"
+    return result
+  end # def sprintf
 end # class LogStash::Event

data/lib/logstash/filters/grep.rb

@@ -67,7 +67,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
     matched = false
     config.each do |match|
       if ! match["match"]
-        @logging.debug(["Skipping match object, no match key", match])
+        @logger.debug(["Skipping match object, no match key", match])
         next
       end
 
@@ -75,9 +75,12 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
       # apply any fields/tags.
       match_count = 0
       match["match"].each do |field, re|
-        next unless event[field]
+        if !event[field]
+          @logger.debug(["Skipping match object, field not present", field, event, event[field]])
+          next
+        end
 
-        if event[field].empty? and match["negate"] == true
+        if event[field].nil? and match["negate"] == true
           match_count += 1
         end
         (event[field].is_a?(Array) ? event[field] : [event[field]]).each do |value|
@@ -86,6 +89,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
             next if re.match(value)
             @logger.debug(["grep not-matched (negate requsted)", { field => value }])
           else
+            @logger.debug(["trying regex", re, value])
             next unless re.match(value)
             @logger.debug(["grep matched", { field => value }])
           end
@@ -101,14 +105,14 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
         if match["add_fields"]
           match["add_fields"].each do |field, value|
             event[field] ||= []
-            event[field] << value
+            event[field] << event.sprintf(value)
             @logger.debug("grep: adding #{value} to field #{field}")
           end
         end # if match["add_fields"]
 
         if match["add_tags"]
           match["add_tags"].each do |tag|
-            event.tags << tag
+            event.tags << event.sprintf(tag)
             @logger.debug("grep: adding tag #{tag}")
           end
         end # if match["add_tags"]

data/lib/logstash/filters/multiline.rb

@@ -17,6 +17,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   # - multiline:
   #     <type>:
   #       pattern: <regexp>
+  #       negate: true
   #       what: next
   #     <type>
   #        pattern: <regexp>
@@ -28,6 +29,10 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   # The 'what' must be "previous" or "next" and indicates the relation
   # to the multi-line event.
   #
+  # The 'negate' can be "true" or "false" (defaults false). If true, a 
+  # message not matching the pattern will constitute a match of the multiline
+  # filter and the what will be applied. (vice-versa is also true)
+  #
   # For example, java stack traces are multiline and usually have the message
   # starting at the far-left, then each subsequent line indented. Do this:
   # 
@@ -96,7 +101,10 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
     key = [event.source, event.type]
     pending = @pending[key]
 
-    @logger.debug(["Reg: ", typeconfig["pattern"], event.message, match])
+    @logger.debug(["Reg: ", typeconfig["pattern"], event.message, match, typeconfig["negate"]])
+    # Add negate option
+    match = (match and !typeconfig["negate"]) || (!match and typeconfig["negate"])
+
     case typeconfig["what"]
     when "previous"
       if match

data/lib/logstash/outputs/elasticsearch.rb

@@ -50,6 +50,7 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
     end
     indexmap_req.errback do
       @logger.warn(["Failure configuring index", @esurl.to_s, indexmap])
+      raise "Failure configuring index: #{@esurl.to_s}"
     end
   end # def register
 
@@ -124,6 +125,7 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
     req.errback do
       $stderr.puts "Request to index to #{@url.to_s} failed (will retry, #{tries} tries left). Event was #{event.to_s}"
       EventMachine::add_timer(2) do
+        # TODO(sissel): Actually abort if we retry too many times.
         receive_http(event, tries - 1)
       end
     end

data/lib/logstash/web/public/js/logstash.js

@@ -10,8 +10,11 @@
       if (query == undefined || query == "") {
         return;
       }
+      //console.log("Searching: " + query);
+
       var display_query = query.replace("<", "&lt;").replace(">", "&gt;")
       $("#querystatus").html("Loading query '" + display_query + "'")
+      //console.log(logstash.params)
       logstash.params.q = query;
       document.location.hash = escape(JSON.stringify(logstash.params));
       $("#results").load("/search/ajax", logstash.params);
@@ -19,12 +22,22 @@
     }, /* search */
 
     parse_params: function(href) {
-      var params = href.replace(/^[^?]*\?/, "").split("&")
-      for (var p in params) {
-        var a = params[p].split("=");
-        var key = a[0]
-        var value = a[1]
-        logstash.params[key] = unescape(value)
+      var query = href.replace(/^[^?]*\?/, "");
+      if (query == href) {
+        //console.log("No query params in link " + href);
+        /* No query params */
+        return {};
+      }
+
+      //console.log({ "query": query });
+      var param_list = query.split("&");
+      params = {};
+      //console.log({ "Parsed params": params });
+      for (var p in param_list) {
+        var a = param_list[p].split("=");
+        var key = a[0];
+        var value = a[1];
+        params[key] = unescape(value);
       }
       return params;
     },
@@ -89,6 +102,7 @@
     } else {
       /* No hash. See if there's a query param. */
       var params = logstash.parse_params(location.href);
+      //console.log(params)
       for (var p in params) {
         logstash.params[p] = params[p];
       }
@@ -104,9 +118,10 @@
       }
     });
 
-    $("a.pager").live("click", function() {
+    $("a.pager, a.querychanger").live("click", function() {
+      /* TODO(sissel): Allow 'control click' and 'middle click' to act normally */
       var href = $(this).attr("href");
-      var params = logstash.parse_params(location.href);
+      var params = logstash.parse_params(href);
       for (var p in params) {
         logstash.params[p] = params[p];
       }
@@ -114,16 +129,6 @@
       return false;
     });
 
-    $("a.querychanger").live("click", function() {
-      var href = $(this).attr("href");
-      var re = new RegExp("[&?]q=([^&]+)");
-      var match = re.exec(href);
-      if (match) {
-        logstash.search(match[1]);
-      }
-      return false;
-    });
-
     var result_row_selector = "table.results tr.event";
     $(result_row_selector).live("click", function() {
       var data = eval($("td.message", this).data("full"));

data/lib/logstash/web/server.rb

@@ -32,7 +32,7 @@ class LogStash::Web::Server < Sinatra::Base
   end # '/'
 
   aget '/search' do
-    result_callback = proc do 
+    result_callback = proc do
       status 500 if @error
 
       params[:format] ||= "html"
@@ -104,7 +104,7 @@ class LogStash::Web::Server < Sinatra::Base
       if count and offset
         if @total > (count + offset)
           @result_end = (count + offset)
-        else 
+        else
           @result_end = @total
         end
         @result_start = offset
@@ -137,27 +137,39 @@ class LogStash::Web::Server < Sinatra::Base
 end # class LogStash::Web::Server
 
 require "optparse"
-Settings = Struct.new(:daemonize, :logfile)
+Settings = Struct.new(:daemonize, :logfile, :address, :port)
 settings = Settings.new
+
+settings.address      = "0.0.0.0"
+settings.port      = 9292
+
 progname = File.basename($0)
 
 opts = OptionParser.new do |opts|
   opts.banner = "Usage: #{progname} [options]"
 
-  opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do 
+  opts.on("-d", "--daemonize", "Daemonize (default is run in foreground).") do
     settings.daemonize = true
   end
 
   opts.on("-l", "--log FILE", "Log to a given path. Default is stdout.") do |path|
     settings.logfile = path
   end
+
+  opts.on("-a", "--address ADDRESS", "Address on which to start webserver. Default is 0.0.0.0.") do |address|
+    settings.address = address
+  end
+
+  opts.on("-p", "--port PORT", "Port on which to start webserver. Default is 9292.") do |port|
+    settings.port = port.to_i
+  end
 end
 
 opts.parse!
 
 if settings.daemonize
   if Process.fork == nil
-    Process.setsid 
+    Process.setsid
   else
     exit(0)
   end
@@ -168,7 +180,7 @@ if settings.logfile
   STDOUT.reopen(logfile)
   STDERR.reopen(logfile)
 elsif settings.daemonize
-  # Write to /dev/null if 
+  # Write to /dev/null if
   devnull = File.open("/dev/null", "w")
   STDOUT.reopen(devnull)
   STDERR.reopen(devnull)
@@ -178,4 +190,4 @@ Rack::Handler::Thin.run(
   Rack::CommonLogger.new( \
     Rack::ShowExceptions.new( \
       LogStash::Web::Server.new)),
-  :Port => 9292)
+  :Port => settings.port, :Host => settings.address)

data/patterns/grok-patterns

@@ -7,6 +7,7 @@ BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
 BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
 
 POSINT \b(?:[0-9]+)\b
+TWODIGITINT [0-9]{2}
 WORD \b\w+\b
 NOTSPACE \S+
 DATA .*?
@@ -88,3 +89,10 @@ QS %{QUOTEDSTRING}
 # Log formats
 SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
 COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
+
+#
+# Custom formats
+# Add additional custom patterns below
+DATESTAMP_RAILS %{DAY} %{MONTH} %{MONTHDAY} %{TIME} (?:%{INT:ZONE} )?%{YEAR}
+DATESTAMP_MYSQL %{TWODIGITINT:year}%{TWODIGITINT:month}%{TWODIGITINT:day}\s+%{TIME}
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-lite
 version: !ruby/object:Gem::Version 
-  hash: 40220244287589
+  hash: 40220406260823
   prerelease: 
   segments: 
   - 0
   - 2
-  - 20110122143801
-  version: 0.2.20110122143801
+  - 20110203130400
+  version: 0.2.20110203130400
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-22 00:00:00 -08:00
+date: 2011-02-03 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -221,7 +221,6 @@ files:
 - examples/sample-agent-in-ruby.rb
 - etc/tograylog.yaml
 - etc/logstash-elasticsearch-rabbitmq-river.yaml
-- etc/foo.yaml
 - etc/init/logstash
 - etc/logstash-nagios.yaml
 - etc/logstash-reader.yaml

data/etc/foo.yaml

@@ -1,6 +0,0 @@
---- 
-inputs:
-  all:
-  - syslog://0.0.0.0:5544/
-outputs:
-- stdout:///