logstash-lite 0.2.20101222161646 → 0.2.20110112115019

data/lib/logstash/filters/grokdiscovery.rb

@@ -1,15 +1,17 @@
 require "logstash/filters/base"
-
+require "logstash/namespace"
 gem "jls-grok", ">=0.2.3071"
 require "grok" # rubygem 'jls-grok'
 
 class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
+  public
   def initialize(config = {})
     super
 
     @discover_fields = {}
   end # def initialize
 
+  public
   def register
     # TODO(sissel): Make patterns files come from the config
     @config.each do |type, typeconfig|
@@ -24,6 +26,7 @@ class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
     end # @config.each
   end # def register
 
+  public
   def filter(event)
     # parse it with grok
     message = event.message

data/lib/logstash/filters/multiline.rb

@@ -4,6 +4,7 @@
 # 
 
 require "logstash/filters/base"
+require "logstash/namespace"
 
 class LogStash::Filters::Multiline < LogStash::Filters::Base
   # The 'date' filter will take a value from your event and use it as the
@@ -46,6 +47,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   #       pattern: /\\$/
   #       what: next
   #
+  public
   def initialize(config = {})
     super
 
@@ -53,6 +55,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
     @pending = Hash.new
   end # def initialize
 
+  public
   def register
     @config.each do |type, typeconfig|
       # typeconfig will be a hash containing 'pattern' and 'what'
@@ -82,10 +85,10 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
         @logger.fatal(["Invalid pattern for multiline filter on type '#{type}'",
                       typeconfig, e])
       end
-
     end # @config.each
   end # def register
 
+  public
   def filter(event)
     return unless @types.member?(event.type)
     typeconfig = @types[event.type]
@@ -147,6 +150,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
   end # def filter
 
   # flush any pending messages
+  public
   def flush(source, type)
     key = [source, type]
     if @pending[key]
@@ -154,5 +158,5 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
       @pending.delete(key)
     end
     return event
-  end
+  end # def flush
 end # class LogStash::Filters::Date

data/lib/logstash/inputs.rb

@@ -1,9 +1,13 @@
-
 require "logstash/namespace"
 require "logstash/ruby_fixes"
 require "uri"
 
 module LogStash::Inputs
+  # Given a URL, try to load the class that supports it.
+  # That is, if we have an input of "foo://blah/" then
+  # we will try to load logstash/inputs/foo and will
+  # expect a class LogStash::Inputs::Foo
+  public
   def self.from_url(url, type, &block)
     # Assume file paths if we start with "/"
     url = "file://#{url}" if url.start_with?("/")
@@ -12,7 +16,7 @@ module LogStash::Inputs
     # TODO(sissel): Add error handling
     # TODO(sissel): Allow plugin paths
     klass = uri.scheme.capitalize
-    file = uri.scheme
+    file = uri.scheme.downcase
     require "logstash/inputs/#{file}"
     LogStash::Inputs.const_get(klass).new(uri, type, &block)
   end # def from_url

data/lib/logstash/inputs/amqp.rb

@@ -1,11 +1,13 @@
-require "logstash/inputs/base"
 require "amqp" # rubygem 'amqp'
+require "logstash/inputs/base"
+require "logstash/namespace"
 require "mq" # rubygem 'amqp'
 require "uuidtools" # rubygem 'uuidtools'
 
 class LogStash::Inputs::Amqp < LogStash::Inputs::Base
   MQTYPES = [ "fanout", "queue", "topic" ]
 
+  public
   def initialize(url, type, config={}, &block)
     super
 
@@ -20,8 +22,9 @@ class LogStash::Inputs::Amqp < LogStash::Inputs::Base
     if !MQTYPES.include?(@mqtype)
       raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.JOIN(", ")}"
     end
-  end
+  end # def initialize
 
+  public
   def register
     @logger.info("Registering input #{@url}")
     @amqp = AMQP.connect(:host => @url.host)

data/lib/logstash/inputs/base.rb

@@ -5,6 +5,8 @@ require "uri"
 
 class LogStash::Inputs::Base
   attr_accessor :logger
+
+  public
   def initialize(url, type, config={}, &block)
     @logger = LogStash::Logger.new(STDERR)
     @url = url
@@ -13,21 +15,32 @@ class LogStash::Inputs::Base
     @callback = block
     @type = type
     @tags = []
-  end
 
+    @urlopts = {}
+    if @url.query
+      @urlopts = CGI.parse(@url.query)
+      @urlopts.each do |k, v|
+        @urlopts[k] = v.last if v.is_a?(Array)
+      end
+    end
+  end # def initialize
+
+  public
   def register
     raise "#{self.class}#register must be overidden"
-  end
+  end # def register
 
+  public
   def tag(newtag)
     @tags << newtag
-  end
+  end # def tag
 
+  public
   def receive(event)
     @logger.debug(["Got event", { :url => @url, :event => event }])
     # Only override the type if it doesn't have one
     event.type = @type if !event.type 
     event.tags |= @tags # set union
     @callback.call(event)
-  end
+  end # def receive
 end # class LogStash::Inputs::Base

data/lib/logstash/inputs/beanstalk.rb

@@ -1,15 +1,18 @@
-require "logstash/inputs/base"
 require "em-jack"
+require "logstash/inputs/base"
+require "logstash/namespace"
 
 class LogStash::Inputs::Beanstalk < LogStash::Inputs::Base
+  public
   def initialize(url, type, config={}, &block)
     super
 
     if @url.path == "" or @url.path == "/"
       raise "must specify a tube for beanstalk output"
     end
-  end
+  end # def initialize
 
+  public
   def register
     tube = @url.path[1..-1] # Skip leading '/'
     port = @url.port || 11300

data/lib/logstash/inputs/file.rb

@@ -1,22 +1,26 @@
-require "logstash/inputs/base"
 require "eventmachine-tail"
+require "logstash/inputs/base"
+require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::File < LogStash::Inputs::Base
+  public
   def initialize(url, type, config={}, &block)
     super
 
     # Hack the hostname into the url.
     # This works since file:// urls don't generally have a host in it.
     @url.host = Socket.gethostname
-  end
+  end # def initialize
 
+  public
   def register
     @logger.info("Registering #{@url}")
     EventMachine::FileGlobWatchTail.new(@url.path, Reader, interval=60,
                                         exclude=[], receiver=self)
   end # def register
 
+  public
   def receive(filetail, event)
     url = @url.clone
     url.path = filetail.path
@@ -31,6 +35,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     @callback.call(event)
   end # def receive
 
+  private
   class Reader < EventMachine::FileTail
     def initialize(path, receiver)
       super(path)

data/lib/logstash/inputs/internal.rb

@@ -1,11 +1,12 @@
-
-require "logstash/inputs/base"
 require "eventmachine-tail"
+require "logstash/inputs/base"
+require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Internal < LogStash::Inputs::Base
   attr_reader :channel
 
+  public
   def initialize(url, type, config={}, &block)
     super
 
@@ -14,6 +15,7 @@ class LogStash::Inputs::Internal < LogStash::Inputs::Base
     @channel = EventMachine::Channel.new
   end
 
+  public
   def register
     @logger.info("Registering input #{@url}")
     @channel.subscribe do |event|
@@ -21,6 +23,7 @@ class LogStash::Inputs::Internal < LogStash::Inputs::Base
     end
   end # def register
 
+  public
   def receive(event)
     if !event.is_a?(LogStash::Event)
       event = LogStash::Event.new({

data/lib/logstash/inputs/stdin.rb

@@ -0,0 +1,38 @@
+require "eventmachine-tail"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+class LogStash::Inputs::Stdin < LogStash::Inputs::Base
+  public
+  def register
+    EventMachine::attach($stdin, InputHandler, self)
+    @url.host = Socket.gethostname
+  end # def register
+
+  public
+  def receive(event)
+    event = LogStash::Event.new({
+      "@message" => event,
+      "@type" => @type,
+      "@tags" => @tags.clone,
+    })
+    event.source = @url
+    @logger.debug(["Got event", event])
+    @callback.call(event)
+  end # def receive
+
+  class InputHandler < EventMachine::Connection
+    def initialize(obj)
+      @receiver = obj
+    end # def initialize
+
+    def receive_data(data)
+      @buffer ||= BufferedTokenizer.new
+      @buffer.extract(data).each do |line|
+        @receiver.receive(line)
+      end
+    end # def receive_data
+  end # class InputHandler
+
+end # class LogStash::Inputs::Stdin

data/lib/logstash/inputs/stomp.rb

@@ -1,8 +1,21 @@
 require "logstash/inputs/base"
+require "logstash/namespace"
 require "logstash/stomp/handler"
 
 class LogStash::Inputs::Stomp < LogStash::Inputs::Base
+  public
+  def initialize(url, config={}, &block)
+    super
+    @logger.debug(["Connecting", { :url => @url }])
+  end # def initialize
+
+  public
+  def register
+    @logger.info(["Registering input", { :url => @url}])
+    EventMachine::connect(@url.host, @url.port, InputHandler, self, @logger, @url)
+  end # def register
 
+  private
   class InputHandler < LogStash::Stomp::Handler
     def receive_msg(message)
       super
@@ -13,15 +26,4 @@ class LogStash::Inputs::Stomp < LogStash::Inputs::Base
       end
     end # def receive_msg
   end # class StompHandler
-
-  def initialize(url, config={}, &block)
-    super
-
-    @logger.debug(["Connecting", { :url => @url }])
-  end # def initialize
-
-  def register
-    @logger.info(["Registering input", { :url => @url}])
-    EventMachine::connect(@url.host, @url.port, InputHandler, self, @logger, @url)
-  end # def register
-end # class LogStash::Inputs::Amqp
+end # class LogStash::Inputs::Stomp

data/lib/logstash/inputs/syslog.rb

@@ -1,11 +1,12 @@
-require "logstash/inputs/base"
-require "eventmachine-tail"
-require "socket" # for Socket.gethostname
 require "date"
+require "eventmachine-tail"
+require "logstash/inputs/base"
+require "logstash/namespace"
 require "logstash/time" # should really use the filters/date.rb bits
-
+require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Syslog < LogStash::Inputs::Base
+  public
   def register
     if !@url.host or !@url.port
       @logger.fatal("No host or port given in #{self.class}: #{@url}")
@@ -26,6 +27,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
       #<priority       timestamp          Mmm dd hh:mm:ss             host  msg
   end # def register
 
+  public
   def receive(host, port, message)
     url = @url.clone
     url.host = host
@@ -48,6 +50,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # If the message cannot be recognized (see @@syslog_re), we'll
   # treat it like the whole event.message is correct and try to fill
   # the missing pieces (host, priority, etc)
+  public
   def syslog_relay(event, url)
     match = @@syslog_re.match(event.message)
     if match
@@ -90,6 +93,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     end
   end # def syslog_relay
 
+  private
   class TCPInput < EventMachine::Connection
     def initialize(receiver, logger)
       @logger = logger
@@ -107,6 +111,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     end # def receive_data
   end # class TCPInput
 
+  private
   class UDPInput < EventMachine::Connection
     def initialize(receiver, logger)
       @logger = logger
@@ -120,4 +125,4 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
       @receiver.receive(host, port, data.chomp)
     end # def receive_data
   end # class UDPInput
-end # class LogStash::Inputs::Tcp
+end # class LogStash::Inputs::Syslog

data/lib/logstash/inputs/tcp.rb

@@ -1,12 +1,15 @@
-require "logstash/inputs/base"
 require "eventmachine-tail"
+require "logstash/inputs/base"
+require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Tcp < LogStash::Inputs::Base
+  public
   def initialize(url, type, config={}, &block)
     super
-  end
+  end # def initialize
 
+  public
   def register
     if !@url.host or !@url.port
       @logger.fatal("No host or port given in #{self.class}: #{@url}")
@@ -16,8 +19,9 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
     @logger.info("Starting tcp listener for #{@url}")
     EventMachine::start_server(@url.host, @url.port, TCPInput, @url, self, @logger)
-  end
+  end # def register
 
+  public
   def receive(host, port, event)
     url = @url.clone
     url.host = host
@@ -33,6 +37,7 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
     @callback.call(event)
   end # def receive
 
+  private
   class TCPInput < EventMachine::Connection
     def initialize(url, receiver, logger)
       @logger = logger

data/lib/logstash/inputs/twitter.rb

@@ -0,0 +1,81 @@
+require "cgi"
+require "em-http-request"
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# TODO(sissel): This could use some refactoring.
+class LogStash::Inputs::Twitter < LogStash::Inputs::Base
+  public
+  def register
+    api_url = "https://stream.twitter.com/1/statuses/filter.json"
+    @http = EventMachine::HttpRequest.new(api_url)
+    @logger.info(["Registering input", { :url => @url, :api_url => api_url, :params => @urlopts }])
+    source = "twitter://...#{@url.path}?#{@url.query}"
+
+    if @url.user.nil? or @user.password.nil?
+      message = "User and password missing for twitter input #{@url.to_s}")
+      @logger.error(message)
+      raise message
+    end
+
+    req = nil
+    connect = proc do
+      @logger.info(["Connecting", { :url => @url, :api_url => api_url, :params => @urlopts}])
+      req = @http.post :body => @urlopts,
+                       :head => { "Authorization" => [ @url.user, @url.password ] }
+      buffer = BufferedTokenizer.new
+
+      req.stream do |chunk|
+        buffer.extract(chunk).each do |line|
+          begin
+            tweet = JSON.parse(line)
+          rescue JSON::ParserError => e
+            @logger.warn("Invalid JSON, aborting connection: #{line}")
+            req.errback
+            next
+          end
+          next if !tweet
+
+          event = LogStash::Event.new({
+            "@message" => tweet["text"],
+            "@type" => @type,
+            "@tags" => @tags.clone,
+          })
+
+          event.fields.merge!(
+            "user" => (tweet["user"]["screen_name"] rescue nil),
+            "client" => (tweet["user"]["source"] rescue nil),
+            "retweeted" => (tweet["retweeted"] rescue nil)
+          )
+
+          event.fields["in-reply-to"] = tweet["in_reply_to_status_id"] if tweet["in_reply_to_status_id"]
+
+          urls = tweet["entities"]["urls"] rescue []
+          if urls.size > 0
+            event.fields["urls"] = urls.collect { |u| u["url"] }
+          end
+
+          event.source = source
+          @logger.debug(["Got event", event])
+          @callback.call(event)
+        end # buffer.extract
+      end # req.stream
+
+      req.errback do
+        @logger.warn(["Error occurred, not sure what, seriously. Reconnecting!", { :url => @url }])
+
+        req.close_connection() rescue nil
+
+        EventMachine::Timer.new(60) do
+          connect.call
+        end
+      end # req.errback
+
+      req.callback do
+        @logger.warn(["Request completed. Unexpected!", { :url => @url }])
+      end
+    end # connect = proc do
+
+    connect.call
+  end # def register
+end # class LogStash::Inputs::Twitter