logstash-lib 1.3.2

data/lib/logstash/inputs/threadable.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/inputs/base"
+
+# This is the threadable class for logstash inputs. 
+# Use this class in your inputs if it can support multiple threads
+class LogStash::Inputs::Threadable < LogStash::Inputs::Base
+
+  # Set this to the number of threads you want this input to spawn.
+  # This is the same as declaring the input multiple times
+  config :threads, :validate => :number, :default => 1
+ 
+  def initialize(params)
+    super
+    @threadable = true
+  end
+
+end # class LogStash::Inputs::Threadable

data/lib/logstash/inputs/twitter.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "json"
+
+# Read events from the twitter streaming api.
+class LogStash::Inputs::Twitter < LogStash::Inputs::Base
+
+  config_name "twitter"
+  milestone 1
+
+  # Your twitter app's consumer key
+  #
+  # Don't know what this is? You need to create an "application"
+  # on twitter, see this url: <https://dev.twitter.com/apps/new>
+  config :consumer_key, :validate => :string, :required => true
+
+  # Your twitter app's consumer secret
+  #
+  # If you don't have one of these, you can create one by
+  # registering a new application with twitter:
+  # <https://dev.twitter.com/apps/new>
+  config :consumer_secret, :validate => :password, :required => true
+
+  # Your oauth token.
+  #
+  # To get this, login to twitter with whatever account you want,
+  # then visit <https://dev.twitter.com/apps>
+  #
+  # Click on your app (used with the consumer_key and consumer_secret settings)
+  # Then at the bottom of the page, click 'Create my access token' which
+  # will create an oauth token and secret bound to your account and that
+  # application.
+  config :oauth_token, :validate => :string, :required => true
+  
+  # Your oauth token secret.
+  #
+  # To get this, login to twitter with whatever account you want,
+  # then visit <https://dev.twitter.com/apps>
+  #
+  # Click on your app (used with the consumer_key and consumer_secret settings)
+  # Then at the bottom of the page, click 'Create my access token' which
+  # will create an oauth token and secret bound to your account and that
+  # application.
+  config :oauth_token_secret, :validate => :password, :required => true
+
+  # Any keywords to track in the twitter stream
+  config :keywords, :validate => :array, :required => true
+
+  public
+  def register
+    require "twitter"
+    @client = Twitter::Streaming::Client.new do |c|
+      c.consumer_key = @consumer_key
+      c.consumer_secret = @consumer_secret.value
+      c.access_token = @oauth_token
+      c.access_token_secret = @oauth_token_secret.value
+    end
+  end
+
+  public
+  def run(queue)
+    @logger.info("Starting twitter tracking", :keywords => @keywords)
+    @client.filter(:track => @keywords.join(",")) do |tweet|
+      @logger.info? && @logger.info("Got tweet", :user => tweet.user.screen_name, :text => tweet.text)
+      event = LogStash::Event.new(
+        "@timestamp" => tweet.created_at.gmtime,
+        "message" => tweet.full_text,
+        "user" => tweet.user.screen_name,
+        "client" => tweet.source,
+        "retweeted" => tweet.retweeted?,
+        "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
+      )
+      decorate(event)
+      event["in-reply-to"] = tweet.in_reply_to_status_id if tweet.reply?
+      unless tweet.urls.empty?
+        event["urls"] = tweet.urls.map(&:expanded_url).map(&:to_s)
+      end
+      queue << event
+    end # client.filter
+  end # def run
+end # class LogStash::Inputs::Twitter

data/lib/logstash/inputs/udp.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Read messages as events over the network via udp.
+#
+class LogStash::Inputs::Udp < LogStash::Inputs::Base
+  config_name "udp"
+  milestone 2
+
+  default :codec, "plain"
+
+  # The address to listen on
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on. Remember that ports less than 1024 (privileged
+  # ports) may require root or elevated privileges to use.
+  config :port, :validate => :number, :required => true
+
+  # Buffer size
+  config :buffer_size, :validate => :number, :default => 8192
+
+  public
+  def initialize(params)
+    super
+    BasicSocket.do_not_reverse_lookup = true
+  end # def initialize
+
+  public
+  def register
+    @udp = nil
+  end # def register
+
+  public
+  def run(output_queue)
+    begin
+      # udp server
+      udp_listener(output_queue)
+    rescue LogStash::ShutdownSignal
+      # do nothing, shutdown was requested.
+    rescue => e
+      @logger.warn("UDP listener died", :exception => e, :backtrace => e.backtrace)
+      sleep(5)
+      retry
+    end # begin
+  end # def run
+
+  private
+  def udp_listener(output_queue)
+    @logger.info("Starting UDP listener", :address => "#{@host}:#{@port}")
+
+    if @udp && ! @udp.closed?
+      @udp.close
+    end
+
+    @udp = UDPSocket.new(Socket::AF_INET)
+    @udp.bind(@host, @port)
+
+    loop do
+      payload, client = @udp.recvfrom(@buffer_size)
+      @codec.decode(payload) do |event|
+        decorate(event)
+        event["host"] = client[3]
+        output_queue << event
+      end
+    end
+  ensure
+    if @udp
+      @udp.close_read rescue nil
+      @udp.close_write rescue nil
+    end
+  end # def udp_listener
+
+  public
+  def teardown
+    @udp.close if @udp && !@udp.closed?
+  end
+
+end # class LogStash::Inputs::Udp

data/lib/logstash/inputs/unix.rb

@@ -0,0 +1,163 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Read events over a UNIX socket.
+#
+# Like stdin and file inputs, each event is assumed to be one line of text.
+#
+# Can either accept connections from clients or connect to a server,
+# depending on `mode`.
+class LogStash::Inputs::Unix < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+  config_name "unix"
+  milestone 2
+
+  default :codec, "line"
+
+  # When mode is `server`, the path to listen on.
+  # When mode is `client`, the path to connect to.
+  config :path, :validate => :string, :required => true
+
+  # Remove socket file in case of EADDRINUSE failure
+  config :force_unlink, :validate => :boolean, :default => false
+
+  # The 'read' timeout in seconds. If a particular connection is idle for
+  # more than this timeout period, we will assume it is dead and close it.
+  #
+  # If you never want to timeout, use -1.
+  config :data_timeout, :validate => :number, :default => -1
+
+  # Mode to operate in. `server` listens for client connections,
+  # `client` connects to a server.
+  config :mode, :validate => ["server", "client"], :default => "server"
+
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    require "socket"
+    require "timeout"
+
+    if server?
+      @logger.info("Starting unix input listener", :address => "#{@path}", :force_unlink => "#{@force_unlink}")
+      begin
+        @server_socket = UNIXServer.new(@path)
+      rescue Errno::EADDRINUSE, IOError
+        if @force_unlink
+          File.unlink(@path)
+          begin
+            @server_socket = UNIXServer.new(@path)
+            return
+          rescue Errno::EADDRINUSE, IOError
+            @logger.error("!!!Could not start UNIX server: Address in use",
+                          :path => @path)
+            raise
+          end
+        end
+        @logger.error("Could not start UNIX server: Address in use",
+                      :path => @path)
+        raise
+      end
+    end
+  end # def register
+
+  private
+  def handle_socket(socket, output_queue)
+    begin
+      hostname = Socket.gethostname
+      loop do
+        buf = nil
+        # NOTE(petef): the timeout only hits after the line is read
+        # or socket dies
+        # TODO(sissel): Why do we have a timeout here? What's the point?
+        if @data_timeout == -1
+          buf = socket.readpartial(16384)
+        else
+          Timeout::timeout(@data_timeout) do
+            buf = socket.readpartial(16384)
+          end
+        end
+        @codec.decode(buf) do |event|
+          decorate(event)
+          event["host"] = hostname
+          event["path"] = @path
+          output_queue << event
+        end
+      end # loop do
+    rescue => e
+      @logger.debug("Closing connection", :path => @path,
+      :exception => e, :backtrace => e.backtrace)
+    rescue Timeout::Error
+      @logger.debug("Closing connection after read timeout",
+      :path => @path)
+    end # begin
+
+  ensure
+    begin
+      socket.close
+    rescue IOError
+      #pass
+    end # begin
+  end
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+  public
+  def run(output_queue)
+    if server?
+      @thread = Thread.current
+      @client_threads = []
+      loop do
+        # Start a new thread for each connection.
+        begin
+          @client_threads << Thread.start(@server_socket.accept) do |s|
+            # TODO(sissel): put this block in its own method.
+
+            @logger.debug("Accepted connection",
+                          :server => "#{@path}")
+            begin
+              handle_socket(s, output_queue)
+            rescue Interrupted
+              s.close rescue nil
+            end
+          end # Thread.start
+        rescue IOError, Interrupted
+          if @interrupted
+            # Intended shutdown, get out of the loop
+            @server_socket.close
+            @client_threads.each do |thread|
+              thread.raise(IOError.new)
+            end
+            break
+          else
+            # Else it was a genuine IOError caused by something else, so propagate it up..
+            raise
+          end
+        end
+      end # loop
+    else
+      loop do
+        client_socket = UNIXSocket.new(@path)
+        client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+        @logger.debug("Opened connection", :client => @path)
+        handle_socket(client_socket, output_queue)
+      end # loop
+    end
+  end # def run
+
+  public
+  def teardown
+    if server?
+      File.unlink(@path)
+      @interrupted = true
+      @thread.raise(Interrupted.new)
+    end
+  end # def teardown
+end # class LogStash::Inputs::Unix

data/lib/logstash/inputs/varnishlog.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+# Read from varnish cache's shared memory log
+class LogStash::Inputs::Varnishlog < LogStash::Inputs::Threadable
+  config_name "varnishlog"
+  milestone 1
+
+  public
+  def register
+    require 'varnish'
+    @vd = Varnish::VSM.VSM_New
+    Varnish::VSL.VSL_Setup(@vd)
+    Varnish::VSL.VSL_Open(@vd, 1)
+
+  end # def register
+
+  def run(queue)
+    @q = queue
+    @hostname = Socket.gethostname
+    Varnish::VSL.VSL_Dispatch(@vd, self.method(:cb).to_proc, FFI::MemoryPointer.new(:pointer))
+  end # def run
+
+  private
+  def cb(priv, tag, fd, len, spec, ptr, bitmap)
+    begin
+      str = ptr.read_string(len)
+      event = LogStash::Event.new("message" => str, "host" => @host)
+      decorate(event)
+      event["varnish_tag"] = tag
+      event["varnish_fd"] = fd
+      event["varnish_spec"] = spec
+      event["varnish_bitmap"] = bitmap
+      @q << event
+    rescue => e
+      @logger.warn("varnishlog exception: #{e.inspect}")
+    ensure
+      return 0
+    end
+  end
+  
+  public
+  def teardown
+    finished
+  end # def teardown
+end # class LogStash::Inputs::Stdin

data/lib/logstash/inputs/websocket.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Read events over the websocket protocol.
+class LogStash::Inputs::Websocket < LogStash::Inputs::Base
+  config_name "websocket"
+  milestone 1
+
+  default :codec, "json"
+
+  # The url to connect to or serve from
+  config :url, :validate => :string, :default => "0.0.0.0"
+
+  # Operate as a client or a server.
+  #
+  # Client mode causes this plugin to connect as a websocket client
+  # to the URL given. It expects to receive events as websocket messages.
+  #
+  # (NOT IMPLEMENTED YET) Server mode causes this plugin to listen on
+  # the given URL for websocket clients. It expects to receive events
+  # as websocket messages from these clients.
+  config :mode, :validate => [ "server", "client" ], :default => "client"
+
+  def register
+    require "ftw"
+  end # def register
+
+  public
+  def run(output_queue)
+    # TODO(sissel): Implement server mode.
+    agent = FTW::Agent.new
+    begin
+      websocket = agent.websocket!(@url)
+      websocket.each do |payload|
+        @codec.decode(payload) do |event|
+          decorate(event)
+          output_queue << event
+        end
+      end
+    rescue => e
+      @logger.warn("websocket input client threw exception, restarting",
+                   :exception => e)
+      sleep(1)
+      retry
+    end # begin
+  end # def run
+
+end # class LogStash::Inputs::Websocket