logstash-lite 0.2.20101201111523 → 0.2.20101207114354

data/bin/logstash

@@ -8,7 +8,7 @@ require "logstash/agent"
 require "optparse"
 require "yaml"
 
-Settings = Struct.new(:config_file, :daemonize)
+Settings = Struct.new(:config_file, :daemonize, :logfile)
 
 settings = Settings.new
 settings.daemonize = false
@@ -23,9 +23,13 @@ opts = OptionParser.new do |opts|
     settings.config_file = arg
   end
 
-  #opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do 
-    #settings.daemonize = true
-  #end
+  opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do 
+    settings.daemonize = true
+  end
+
+  opts.on("-l", "--log FILE", "Log to a given path. Default is stdout.") do |path|
+    settings.logfile = path
+  end
 end
 
 begin
@@ -33,7 +37,6 @@ begin
   if settings.config_file == "" or settings.config_file == nil
     raise "No config file given. (missing -f or --config flag?)"
   end
-
 rescue
   $stderr.puts "#{progname}: #{$!}"
   $stderr.puts opts
@@ -52,5 +55,15 @@ rescue => e
   exit 1
 end
 
+if settings.daemonize
+  if Process.fork == nil
+    Process.setsid 
+  else
+    exit(0)
+  end
+end
 agent = LogStash::Agent.new(config)
-agent.run
+if settings.logfile
+  agent.log_to(settings.logfile)
+end
+agent.run 

data/etc/logstash-grep.yaml

@@ -0,0 +1,31 @@
+---
+inputs:
+  linux-syslog:
+  - /var/log/messages
+filters:
+- grok:
+    linux-syslog: # for logs of type 'linux-syslog'
+      patterns:
+      - %{SYSLOGLINE}
+- date:
+    linux-syslog:
+      timestamp: "%b %e %H:%M:%S"
+      timestamp8601: ISO8601
+- grep:
+    linux-syslog:
+      - match:
+          message: test
+        add_fields:
+          nagios_alert: test_alert
+        add_tags:
+          - nagios
+          - test
+      - match:
+          message: (?i)foo.*bar
+          program: test
+        add_fields:
+          nagios_alert: foo_alert
+        add_tags:
+          - nagios
+outputs:
+- stdout:///

data/etc/logstash-shipper.yaml

@@ -15,4 +15,5 @@ inputs:
   unknown:
   - /b/randomdata
 outputs:
-- amqp://localhost/topic/rawlogs
+#- amqp://localhost/topic/rawlogs
+- websocket://0.0.0.0:3232/

data/etc/logstash-stomp-input.yaml

@@ -0,0 +1,7 @@
+--- 
+inputs:
+  stomp:
+    - stomp://logs:password@localhost:6163/topic/logs
+outputs:
+- stdout:///
+

data/etc/logstash-stomp.yaml

@@ -0,0 +1,7 @@
+--- 
+inputs:
+  tail-syslog:
+  - /var/log/syslog
+outputs:
+- stomp://logs:password@localhost:6163/topic/logs
+

data/lib/logstash/agent.rb

@@ -14,7 +14,7 @@ class LogStash::Agent
   attr_reader :filters
 
   def initialize(config)
-    @logger = LogStash::Logger.new(STDERR)
+    log_to(STDERR)
 
     @config = config
     @outputs = []
@@ -26,6 +26,11 @@ class LogStash::Agent
     # - where to ship to
   end # def initialize
 
+  public
+  def log_to(target)
+    @logger = LogStash::Logger.new(target)
+  end # def log_to
+
   # Register any event handlers with EventMachine
   # Technically, this agent could listen for anything (files, sockets, amqp,
   # stomp, etc).
@@ -55,6 +60,7 @@ class LogStash::Agent
         urls.each do |url|
           @logger.debug("Using input #{url} of type #{type}")
           input = LogStash::Inputs.from_url(url, type) { |event| receive(event) }
+          input.logger = @logger
           input.register
           @inputs << input
         end
@@ -67,6 +73,7 @@ class LogStash::Agent
         name, value = filter
         @logger.debug("Using filter #{name} => #{value.inspect}")
         filter = LogStash::Filters.from_name(name, value)
+        filter.logger = @logger
         filter.register
         @filters << filter
       end # each filter
@@ -76,6 +83,7 @@ class LogStash::Agent
       @config["outputs"].each do |url|
         @logger.debug("Using output #{url}")
         output = LogStash::Outputs.from_url(url)
+        output.logger = @logger
         output.register
         @outputs << output
       end # each output
@@ -135,6 +143,10 @@ class LogStash::Agent
       @sigchannel.push(:USR1)
     end
 
+    Signal.trap("INT") do
+      @sigchannel.push(:INT)
+    end
+
     @sigchannel.subscribe do |msg|
       case msg
       when :USR1
@@ -153,7 +165,12 @@ class LogStash::Agent
         counts.sort { |a,b| a[1] <=> b[1] or a[0] <=> b[0] }.each do |key, value|
           @logger.info("Class: [#{value}] #{key}")
         end
-      end
-    end
-  end
+      when :INT
+        @logger.warn("SIGINT received. Shutting down.")
+        EventMachine::stop_event_loop
+        # TODO(sissel): Should have input/output/filter register shutdown
+        # hooks.
+      end # case msg
+    end # @sigchannel.subscribe
+  end # def sighandler
 end # class LogStash::Components::Agent

data/lib/logstash/filters/base.rb

@@ -2,6 +2,7 @@ require "logstash/namespace"
 require "logstash/logging"
 
 class LogStash::Filters::Base
+  attr_accessor :logger
   def initialize(config = {})
     @logger = LogStash::Logger.new(STDERR)
     @config = config

data/lib/logstash/filters/grep.rb

@@ -0,0 +1,92 @@
+require "logstash/filters/base"
+
+class LogStash::Filters::Grep < LogStash::Filters::Base
+  def initialize(config = {})
+    super
+
+    @config = config
+  end # def initialize
+
+  def register
+    @config.each do |type, matches|
+      if ! matches.is_a?(Array)
+        @logger.warn("grep: #{type} misconfigured; must be an array")
+        next
+      end
+
+      matches.each_index do |i|
+        match = matches[i]
+        if ! match.member?("match")
+          @logger.warn(["grep: #{type}/#{i}: no 'match' section defined", match])
+          next
+        end
+        match["match"].each do |field, re_str|
+          re = Regexp.new(re_str)
+          @config[type][i]["match"][field] = re
+          @logger.debug(["grep: #{type}/#{i}/#{field}", re_str, re])
+        end
+      end # matches.each
+    end # @config.each
+  end # def register
+
+  def filter(event)
+    config = @config[event.type]
+    if not config
+      @logger.debug("grep: skipping type #{event.type} from #{event.source}")
+      return
+    end
+
+    @logger.debug(["Running grep filter", event, config])
+    matched = false
+    config.each do |match|
+      if ! match["match"]
+        @logging.debug(["Skipping match object, no match key", match])
+        next
+      end
+
+      # For each match object, we have to match everything in order to
+      # apply any fields/tags.
+      match_count = 0
+      match["match"].each do |field, re|
+        next unless event[field]
+
+        event[field].each do |value|
+          next unless re.match(value)
+          @logger.debug("grep matched on field #{field}")
+          match_count += 1
+        end
+      end # match["match"].each
+
+      if match_count == match["match"].length
+        matched = true
+        @logger.debug("matched all fields (#{match_count})")
+
+        if match["add_fields"]
+          match["add_fields"].each do |field, value|
+            event[field] ||= []
+            event[field] << value
+            @logger.debug("grep: adding #{value} to field #{field}")
+          end
+        end # if match["add_fields"]
+
+        if match["add_tags"]
+          match["add_tags"].each do |tag|
+            event.tags << tag
+            @logger.debug("grep: adding tag #{tag}")
+          end
+        end # if match["add_tags"]
+      else
+        @logger.debug("match block failed " \
+                      "(#{match_count}/#{match["match"].length} matches)")
+      end # match["match"].each
+    end # config.each
+
+    if not matched
+      @logger.debug("grep: dropping event, no matches")
+      event.cancel
+      return
+    end
+
+    @logger.debug(["Event after grep filter", event.to_hash])
+  end # def filter
+end # class LogStash::Filters::Grep

data/lib/logstash/inputs/base.rb

@@ -4,6 +4,7 @@ require "logstash/logging"
 require "uri"
 
 class LogStash::Inputs::Base
+  attr_accessor :logger
   def initialize(url, type, config={}, &block)
     @logger = LogStash::Logger.new(STDERR)
     @url = url

data/lib/logstash/inputs/stomp.rb

@@ -0,0 +1,27 @@
+require "logstash/inputs/base"
+require "logstash/stomp/handler"
+
+class LogStash::Inputs::Stomp < LogStash::Inputs::Base
+
+  class InputHandler < LogStash::Stomp::Handler
+    def receive_msg(message)
+      super
+
+      unless message.command == "CONNECTED"
+        event = LogStash::Event.from_json(message.body)
+        @input.receive(event)
+      end
+    end # def receive_msg
+  end # class StompHandler
+
+  def initialize(url, config={}, &block)
+    super
+
+    @logger.debug(["Connecting", { :url => @url }])
+  end # def initialize
+
+  def register
+    @logger.info(["Registering input", { :url => @url}])
+    EventMachine::connect(@url.host, @url.port, InputHandler, self, @logger, @url)
+  end # def register
+end # class LogStash::Inputs::Amqp

data/lib/logstash/outputs/base.rb

@@ -5,6 +5,7 @@ require "cgi"
 require "uri"
 
 class LogStash::Outputs::Base
+  attr_accessor :logger
   def initialize(url, config={}, &block)
     @url = url
     @url = URI.parse(url) if url.is_a? String

data/lib/logstash/outputs/stomp.rb

@@ -0,0 +1,22 @@
+require "logstash/outputs/base"
+require "logstash/stomp/handler"
+
+class LogStash::Outputs::Stomp < LogStash::Outputs::Base
+  attr_reader :url
+
+  def initialize(url, config={}, &block)
+    super
+
+    @logger.debug(["Initialize", { :url => @url }])
+  end # def initialize
+
+  def register
+    @logger.info(["Registering output", { :url => @url }])
+    @connection = EventMachine::connect(@url.host, @url.port, LogStash::Stomp::Handler, self, @logger, @url)
+  end # def register
+
+  def receive(event)
+    @logger.debug(["Sending event", { :url => @url, :event => event }])
+    @connection.send(@url.path, event.to_json)
+  end # def receive
+end # class LogStash::Outputs::Stomp

data/lib/logstash/outputs/websocket.rb

@@ -9,10 +9,10 @@ class LogStash::Outputs::Websocket < LogStash::Outputs::Base
   def register
     @channel = EventMachine::Channel.new
     @subscribers = 0
-    host = (@url.host or "0.0.0.0")
-    port = (@url.port or 3000)
+    @url.host = (@url.host or "0.0.0.0")
+    @url.port = (@url.port or 3232)
     @logger.info("Registering websocket on #{@url}")
-    EventMachine::WebSocket.start(:host => host, :port => port) do |ws|
+    EventMachine::WebSocket.start(:host => @url.host, :port => @url.port) do |ws|
       ws.onopen do
         @subscribers += 1
         @logger.info("New #{self.class.name} connection")

data/lib/logstash/stomp/handler.rb

@@ -0,0 +1,34 @@
+# Base of Stomp Handler
+# it handles connecting and subscribing to the stomp broker which
+# is used in both stomp input and output
+class LogStash::Stomp
+  class Handler < EventMachine::Connection
+    include EM::Protocols::Stomp
+
+    def initialize(*args)
+      super
+
+      @input = args[0]
+      @logger = args[1]
+      @url = args[2]
+    end # def initialize
+
+    def connection_completed
+      @logger.debug("Connected")
+      connect :login => @url.user, :passcode => @url.password
+    end # def connection_completed
+
+    def unbind
+      @logger.error(["Error when connecting to stomp broker", { :url => @url }])
+    end # def unbind
+
+    def receive_msg(message)
+      @logger.debug(["receiving message", { :msg => message }])
+      if message.command == "CONNECTED"
+        @logger.debug(["subscribing to", { :path => @url.path }])
+        subscribe @url.path
+        return
+      end
+    end # def receive_msg
+  end # class Handler
+end # class LogStash::Stomp

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-lite
 version: !ruby/object:Gem::Version 
-  hash: 40202402223057
+  hash: 40202414228723
   prerelease: false
   segments: 
   - 0
   - 2
-  - 20101201111523
-  version: 0.2.20101201111523
+  - 20101207114354
+  version: 0.2.20101207114354
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-01 00:00:00 -08:00
+date: 2010-12-07 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -94,6 +94,7 @@ files:
 - lib/logstash/inputs/file.rb
 - lib/logstash/inputs/base.rb
 - lib/logstash/inputs/amqp.rb
+- lib/logstash/inputs/stomp.rb
 - lib/logstash/inputs/tcp.rb
 - lib/logstash/outputs/gelf.rb
 - lib/logstash/outputs/elasticsearch.rb
@@ -103,14 +104,17 @@ files:
 - lib/logstash/outputs/websocket.rb
 - lib/logstash/outputs/base.rb
 - lib/logstash/outputs/amqp.rb
+- lib/logstash/outputs/stomp.rb
 - lib/logstash/outputs/tcp.rb
 - lib/logstash/namespace.rb
 - lib/logstash/time.rb
 - lib/logstash/filters.rb
 - lib/logstash/outputs.rb
+- lib/logstash/stomp/handler.rb
 - lib/logstash/filters/grokdiscovery.rb
 - lib/logstash/filters/multiline.rb
 - lib/logstash/filters/grok.rb
+- lib/logstash/filters/grep.rb
 - lib/logstash/filters/base.rb
 - lib/logstash/filters/field.rb
 - lib/logstash/filters/date.rb
@@ -208,6 +212,7 @@ files:
 - etc/tograylog.yaml
 - etc/logstash-elasticsearch-rabbitmq-river.yaml
 - etc/logstash-reader.yaml
+- etc/logstash-stomp-input.yaml
 - etc/logstash-parser.yaml
 - etc/logstash-mongodb-storage.yaml
 - etc/logstash-standalone.yaml
@@ -216,7 +221,9 @@ files:
 - etc/redhat/logstash.spec
 - etc/redhat/logstash
 - etc/redhat/logstash-agent
+- etc/logstash-stomp.yaml
 - etc/prod.yaml
+- etc/logstash-grep.yaml
 - etc/logstash-shipper.yaml
 - patterns/linux-syslog
 - patterns/haproxy