logstash-lite 0.2.20101222161646 → 0.2.20110112115019

data/bin/logstash

@@ -68,8 +68,19 @@ if settings.daemonize
   end
 end
 
-agent = LogStash::Agent.new(config)
 if settings.logfile
-  agent.log_to(settings.logfile)
+  logfile = File.open(settings.logfile, "w")
+  STDOUT.reopen(logfile)
+  STDERR.reopen(logfile)
+elsif settings.daemonize
+  # Write to /dev/null if 
+  devnull = File.open("/dev/null", "w")
+  STDOUT.reopen(devnull)
+  STDERR.reopen(devnull)
 end
+
+agent = LogStash::Agent.new(config)
+#if settings.logfile
+  #agent.log_to(settings.logfile)
+#end
 agent.run 

data/bin/logstash-test

@@ -60,9 +60,7 @@ def check_libraries
 end
 
 def run_tests
-  require "logstash/test_syntax"
-  require "logstash/filters/test_date"
-  require "logstash/filters/test_multiline"
+  require "#{File.dirname(__FILE__)}/../test/run"
   return Test::Unit::AutoRunner.run
 end
 

data/lib/logstash.rb

@@ -1,3 +1,3 @@
-require "logstash/namespace"
 require "logstash/agent"
 require "logstash/event"
+require "logstash/namespace"

data/lib/logstash/agent.rb

@@ -13,6 +13,7 @@ class LogStash::Agent
   attr_reader :outputs
   attr_reader :filters
 
+  public
   def initialize(config)
     log_to(STDERR)
 
@@ -44,28 +45,26 @@ class LogStash::Agent
     end
 
     # Register input and output stuff
-    if @config.include?("inputs")
-      inputs = @config["inputs"]
-      inputs.each do |value|
-        # If 'url' is an array, then inputs is a hash and the key is the type
-        if inputs.is_a?(Hash)
-          type, urls = value
-        else
-          raise "config error, no type for url #{urls.inspect}"
-        end
+    inputs = @config["inputs"]
+    inputs.each do |value|
+      # If 'url' is an array, then inputs is a hash and the key is the type
+      if inputs.is_a?(Hash)
+        type, urls = value
+      else
+        raise "config error, no type for url #{urls.inspect}"
+      end
 
-        # url could be a string or an array.
-        urls = [urls] if !urls.is_a?(Array)
+      # url could be a string or an array.
+      urls = [urls] if !urls.is_a?(Array)
 
-        urls.each do |url|
-          @logger.debug("Using input #{url} of type #{type}")
-          input = LogStash::Inputs.from_url(url, type) { |event| receive(event) }
-          input.logger = @logger
-          input.register
-          @inputs << input
-        end
-      end # each input
-    end
+      urls.each do |url|
+        @logger.debug("Using input #{url} of type #{type}")
+        input = LogStash::Inputs.from_url(url, type) { |event| receive(event) }
+        input.logger = @logger
+        input.register
+        @inputs << input
+      end
+    end # each input
 
     if @config.include?("filters")
       filters = @config["filters"]
@@ -77,20 +76,18 @@ class LogStash::Agent
         filter.register
         @filters << filter
       end # each filter
-    end
-
-    if @config.include?("outputs")
-      @config["outputs"].each do |url|
-        @logger.debug("Using output #{url}")
-        output = LogStash::Outputs.from_url(url)
-        output.logger = @logger
-        output.register
-        @outputs << output
-      end # each output
-    end
+    end # if we have filters
+
+    @config["outputs"].each do |url|
+      @logger.debug("Using output #{url}")
+      output = LogStash::Outputs.from_url(url)
+      output.logger = @logger
+      output.register
+      @outputs << output
+    end # each output
 
     # Register any signal handlers
-    sighandler
+    register_signal_handler
   end # def register
 
   public
@@ -106,16 +103,16 @@ class LogStash::Agent
     # TODO(sissel): Stop inputs, fluch outputs, wait for finish,
     # then stop the event loop
     EventMachine.stop_event_loop
-  end
+
+    # EventMachine has no default way to indicate a 'stopping' state.
+    $EVENTMACHINE_STOPPING = true
+  end # def stop
 
   protected
   def filter(event)
     @filters.each do |f|
-      # TODO(sissel): Add ability for a filter to cancel/drop a message
       f.filter(event)
-      if event.cancelled?
-        break
-      end
+      break if event.cancelled?
     end
   end # def filter
 
@@ -137,7 +134,7 @@ class LogStash::Agent
   end # def input
 
   public
-  def sighandler
+  def register_signal_handler
     @sigchannel = EventMachine::Channel.new
     Signal.trap("USR1") do
       @sigchannel.push(:USR1)
@@ -172,5 +169,5 @@ class LogStash::Agent
         # hooks.
       end # case msg
     end # @sigchannel.subscribe
-  end # def sighandler
-end # class LogStash::Components::Agent
+  end # def register_signal_handler
+end # class LogStash::Agent

data/lib/logstash/event.rb

@@ -1,9 +1,11 @@
 require "json"
 require "logstash/time"
+require "logstash/namespace"
 require "uri"
 
 # General event type. Will expand this in the future.
-module LogStash; class Event
+class LogStash::Event
+  public
   def initialize(data=Hash.new)
     @cancelled = false
     @data = {
@@ -18,25 +20,31 @@ module LogStash; class Event
     end
   end # def initialize
 
+  public
   def self.from_json(json)
-    return Event.new(JSON.parse(json))
+    return LogStash::Event.new(JSON.parse(json))
   end # def self.from_json
 
+  public
   def cancel
     @cancelled = true
   end
 
+  public
   def cancelled?
     return @cancelled
   end
 
+  public
   def to_s
     return "#{timestamp} #{source}: #{message}"
   end # def to_s
 
+  public
   def timestamp; @data["@timestamp"]; end # def timestamp
   def timestamp=(val); @data["@timestamp"] = val; end # def timestamp=
 
+  public
   def source; @data["@source"]; end # def source
   def source=(val) 
     if val.is_a?(URI)
@@ -48,31 +56,38 @@ module LogStash; class Event
     end
   end # def source=
 
+  public
   def message; @data["@message"]; end # def message
   def message=(val); @data["@message"] = val; end # def message=
 
+  public
   def type; @data["@type"]; end # def type
   def type=(val); @data["@type"] = val; end # def type=
 
+  public
   def tags; @data["@tags"]; end # def tags
   def tags=(val); @data["@tags"] = val; end # def tags=
 
   # field-related access
+  public
   def [](key); @data["@fields"][key] end # def []
   def []=(key, value); @data["@fields"][key] = value end # def []=
   def fields; return @data["@fields"] end # def fields
   
+  public
   def to_json; return @data.to_json end # def to_json
-
   def to_hash; return @data end # def to_hash
 
+  public
   def overwrite(event)
     @data = event.to_hash
   end
 
+  public
   def include?(key); return @data.include?(key) end
 
   # Append an event to this one.
+  public
   def append(event)
     self.message += "\n" + event.message 
     self.tags |= event.tags
@@ -87,5 +102,5 @@ module LogStash; class Event
         self.fields[name] = value
       end
     end # event.fields.each
-  end
-end; end # class LogStash::Event
+  end # def append
+end # class LogStash::Event

data/lib/logstash/filters.rb

@@ -1,7 +1,7 @@
-
 require "logstash/namespace"
 
 module LogStash::Filters
+  public
   def self.from_name(name, *args)
     # TODO(sissel): Add error handling
     # TODO(sissel): Allow plugin paths

data/lib/logstash/filters/base.rb

@@ -3,24 +3,29 @@ require "logstash/logging"
 
 class LogStash::Filters::Base
   attr_accessor :logger
+
+  public
   def initialize(config = {})
     @logger = LogStash::Logger.new(STDERR)
     @config = config
   end # def initialize
 
+  public
   def register
     raise "#{self.class}#register must be overidden"
   end # def register
 
+  public
   def filter(event)
     raise "#{self.class}#filter must be overidden"
   end # def filter
 
+  public
   def add_config(type, typeconfig)
     if @config.include?(type)
       @config[type].merge!(typeconfig)
     else
       @config[type] = typeconfig
     end
-  end
+  end # def add_config
 end # class LogStash::Filters::Base

data/lib/logstash/filters/date.rb

@@ -1,4 +1,5 @@
 require "logstash/filters/base"
+require "logstash/namespace"
 require "logstash/time"
 
 class LogStash::Filters::Date < LogStash::Filters::Base
@@ -16,12 +17,14 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   #       <fieldname>: <format>
   #
   # The format is whatever is supported by Ruby's DateTime.strptime
+  public
   def initialize(config = {})
     super
 
     @types = Hash.new { |h,k| h[k] = [] }
   end # def initialize
 
+  public
   def register
     @config.each do |type, typeconfig|
       @logger.debug "Setting type #{type.inspect} to the config #{typeconfig.inspect}"
@@ -30,6 +33,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
     end # @config.each
   end # def register
 
+  public
   def filter(event)
     @logger.debug "DATE FILTER: received event of type #{event.type}"
     return unless @types.member?(event.type)

data/lib/logstash/filters/field.rb

@@ -1,4 +1,5 @@
 require "logstash/filters/base"
+require "logstash/namespace"
 require "ostruct"
 
 class LogStash::Filters::Field < LogStash::Filters::Base
@@ -8,14 +9,12 @@ class LogStash::Filters::Field < LogStash::Filters::Base
     end
   end
 
-  def initialize(config = {})
-    super
-  end # def initialize
-
+  public
   def register
     # nothing to do
   end # def register
 
+  public
   def filter(event)
     data = EvalSpace.new(event.to_hash)
 
@@ -25,5 +24,5 @@ class LogStash::Filters::Field < LogStash::Filters::Base
       end
     end
     event.cancel
-  end
+  end # def filter
 end # class LogStash::Filters::Field

data/lib/logstash/filters/grep.rb

@@ -1,12 +1,37 @@
 require "logstash/filters/base"
+require "logstash/namespace"
 
+# Grep filter.
+#
+# Useful for:
+# * Dropping events
+# * Tagging events
+# * Adding static fields
+#
+# Events not matched ar dropped. If 'negate' is set to true (defaults false), then
+# matching events are dropped.
+#
+# Config:
+# - grep:
+#   <type>:
+#     - match:
+#         <field>: <regexp>
+#       negate: true/false
+#       add_fields:
+#         <field>: <value>
+#       add_tags:
+#         - tag1
+#         - tag2
+#
 class LogStash::Filters::Grep < LogStash::Filters::Base
+  public
   def initialize(config = {})
     super
 
     @config = config
   end # def initialize
 
+  public
   def register
     @config.each do |type, matches|
       if ! matches.is_a?(Array)
@@ -29,6 +54,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
     end # @config.each
   end # def register
 
+  public
   def filter(event)
     config = @config[event.type]
     if not config
@@ -37,7 +63,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
       return
     end
 
-    @logger.debug(["Running grep filter", event, config])
+    @logger.debug(["Running grep filter", event.to_hash, config])
     matched = false
     config.each do |match|
       if ! match["match"]
@@ -51,9 +77,18 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
       match["match"].each do |field, re|
         next unless event[field]
 
+        if event[field].empty? and match["negate"] == true
+          match_count += 1
+        end
         event[field].each do |value|
-          next unless re.match(value)
-          @logger.debug("grep matched on field #{field}")
+          if match["negate"] == true
+            @logger.debug("want negate match")
+            next if re.match(value)
+            @logger.debug(["grep not-matched (negate requsted)", { field => value }])
+          else
+            next unless re.match(value)
+            @logger.debug(["grep matched", { field => value }])
+          end
           match_count += 1
           break
         end
@@ -77,6 +112,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
             @logger.debug("grep: adding tag #{tag}")
           end
         end # if match["add_tags"]
+
       else
         @logger.debug("match block failed " \
                       "(#{match_count}/#{match["match"].length} matches)")

data/lib/logstash/filters/grok.rb

@@ -1,15 +1,18 @@
 require "logstash/filters/base"
+require "logstash/namespace"
 
 gem "jls-grok", ">=0.2.3071"
 require "grok" # rubygem 'jls-grok'
 
 class LogStash::Filters::Grok < LogStash::Filters::Base
+  public
   def initialize(config = {})
     super
 
     @grokpiles = {}
   end # def initialize
 
+  public
   def register
     # TODO(sissel): Make patterns files come from the config
     @config.each do |type, typeconfig|
@@ -27,6 +30,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     end # @config.each
   end # def register
 
+  public
   def filter(event)
     # parse it with grok
     message = event.message
@@ -38,9 +42,10 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
         pile = @grokpiles[event.type]
         grok, match = pile.match(message)
       end # @grokpiles.include?(event.type)
-     else
-      return
+      # TODO(2.0): support grok pattern discovery
+    else
       @logger.info("Unknown type for #{event.source} (type: #{event.type})")
+      @logger.debug(event.to_hash)
     end
 
     if match
@@ -60,7 +65,9 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
           event.fields[key] = []
         end
 
-        event.fields[key] << value
+        if value && !value.empty?
+          event.fields[key] << value
+        end
       end
     else
       # Tag this event if we can't parse it. We can use this later to