logstash-lib 1.3.2

data/lib/logstash/outputs/websocket.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+
+# This output runs a websocket server and publishes any 
+# messages to all connected websocket clients.
+#
+# You can connect to it with ws://<host\>:<port\>/
+#
+# If no clients are connected, any messages received are ignored.
+class LogStash::Outputs::WebSocket < LogStash::Outputs::Base
+  config_name "websocket"
+  milestone 1
+
+  # The address to serve websocket data from
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to serve websocket data from
+  config :port, :validate => :number, :default => 3232
+
+  public
+  def register
+    require "ftw"
+    require "logstash/outputs/websocket/app"
+    require "logstash/outputs/websocket/pubsub"
+    @pubsub = LogStash::Outputs::WebSocket::Pubsub.new
+    @pubsub.logger = @logger
+    @server = Thread.new(@pubsub) do |pubsub|
+      begin
+        Rack::Handler::FTW.run(LogStash::Outputs::WebSocket::App.new(pubsub, @logger),
+                               :Host => @host, :Port => @port)
+      rescue => e
+        @logger.error("websocket server failed", :exception => e)
+        sleep 1
+        retry
+      end
+    end
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    @pubsub.publish(event.to_json)
+  end # def receive
+
+end # class LogStash::Outputs::Websocket

data/lib/logstash/outputs/websocket/app.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/websocket"
+require "sinatra/base"
+require "rack/handler/ftw" # from ftw
+require "ftw/websocket/rack" # from ftw
+
+class LogStash::Outputs::WebSocket::App < Sinatra::Base
+  def initialize(pubsub, logger)
+    @pubsub = pubsub
+    @logger = logger
+  end
+
+  set :reload_templates, false
+
+  get "/" do
+    # TODO(sissel): Support filters/etc.
+    ws = ::FTW::WebSocket::Rack.new(env)
+    @logger.debug("New websocket client")
+    stream(:keep_open) do |out|
+      @pubsub.subscribe do |event|
+        ws.publish(event)
+      end # pubsub
+    end # stream
+
+    ws.rack_response
+  end # get /
+end # class LogStash::Outputs::WebSocket::App
+

data/lib/logstash/outputs/websocket/pubsub.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/websocket"
+
+class LogStash::Outputs::WebSocket::Pubsub
+  attr_accessor :logger
+
+  def initialize
+    @subscribers = []
+    @subscribers_lock = Mutex.new
+  end # def initialize
+
+  def publish(object)
+    @subscribers_lock.synchronize do
+      break if @subscribers.size == 0
+
+      failed = []
+      @subscribers.each do |subscriber|
+        begin
+          subscriber.call(object)
+        rescue => e
+          @logger.error("Failed to publish to subscriber", :subscriber => subscriber, :exception => e)
+          failed << subscriber
+        end
+      end
+
+      failed.each do |subscriber|
+        @subscribers.delete(subscriber)
+      end
+    end # @subscribers_lock.synchronize
+  end # def Pubsub
+
+  def subscribe(&block)
+    queue = Queue.new
+    @subscribers_lock.synchronize do
+      @subscribers << proc do |event|
+        queue << event
+      end
+    end
+
+    while true
+      block.call(queue.pop)
+    end
+  end # def subscribe
+end # class LogStash::Outputs::WebSocket::Pubsub

data/lib/logstash/outputs/xmpp.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# This output allows you ship events over XMPP/Jabber.
+#
+# This plugin can be used for posting events to humans over XMPP, or you can
+# use it for PubSub or general message passing for logstash to logstash.
+class LogStash::Outputs::Xmpp < LogStash::Outputs::Base
+  config_name "xmpp"
+  milestone 2
+
+  # The user or resource ID, like foo@example.com.
+  config :user, :validate => :string, :required => :true
+
+  # The xmpp password for the user/identity.
+  config :password, :validate => :password, :required => :true
+
+  # The users to send messages to
+  config :users, :validate => :array
+
+  # if muc/multi-user-chat required, give the name of the room that
+  # you want to join: room@conference.domain/nick
+  config :rooms, :validate => :array
+
+  # The xmpp server to connect to. This is optional. If you omit this setting,
+  # the host on the user/identity is used. (foo.com for user@foo.com)
+  config :host, :validate => :string
+
+  # The message to send. This supports dynamic strings like %{host}
+  config :message, :validate => :string, :required => true
+
+  public
+  def register
+    require "xmpp4r"
+    @client = connect
+
+    @mucs = []
+    @users = [] if !@users
+
+    # load the MUC Client if we are joining rooms.
+    if @rooms && !@rooms.empty?
+      require 'xmpp4r/muc'
+      @rooms.each do |room| # handle muc messages in different rooms
+        muc = Jabber::MUC::MUCClient.new(@client)
+        muc.join(room)
+        @mucs << muc
+      end # @rooms.each
+    end # if @rooms
+  end # def register
+
+  public
+  def connect
+    Jabber::debug = true
+    client = Jabber::Client.new(Jabber::JID.new(@user))
+    client.connect(@host)
+    client.auth(@password.value)
+    return client
+  end # def connect
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    string_message = event.sprintf(@message)
+    @users.each do |user|
+      msg = Jabber::Message.new(user, string_message)
+      msg.type = :chat
+      @client.send(msg)
+    end # @targets.each
+
+    msg = Jabber::Message.new(nil, string_message)
+    msg.type = :groupchat
+    @mucs.each do |muc|
+      muc.send(msg)
+    end # @mucs.each
+  end # def receive
+end # class LogStash::Outputs::Xmpp

data/lib/logstash/outputs/zabbix.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+ 
+# The zabbix output is used for sending item data to zabbix via the
+# zabbix_sender executable.
+#
+# For this output to work, your event must have the following fields:
+#
+# * "zabbix_host"    (the host configured in Zabbix)
+# * "zabbix_item"    (the item key on the host in Zabbix)
+#
+# In Zabbix, create your host with the same name (no spaces in the name of 
+# the host supported) and create your item with the specified key as a
+# Zabbix Trapper item.
+#
+# The easiest way to use this output is with the grep filter.
+# Presumably, you only want certain events matching a given pattern
+# to send events to zabbix, so use grep to match and also to add the required
+# fields.
+#
+#      filter {
+#        grep {
+#          type => "linux-syslog"
+#          match => [ "@message", "(error|ERROR|CRITICAL)" ]
+#          add_tag => [ "zabbix-sender" ]
+#          add_field => [
+#            "zabbix_host", "%{source_host}",
+#            "zabbix_item", "item.key"
+#          ]
+#       }
+#     }
+#      
+#     output {
+#       zabbix {
+#         # only process events with this tag
+#         tags => "zabbix-sender"
+#  
+#         # specify the hostname or ip of your zabbix server
+#         # (defaults to localhost)
+#         host => "localhost"
+#  
+#         # specify the port to connect to (default 10051)
+#         port => "10051"
+#  
+#         # specify the path to zabbix_sender
+#         # (defaults to "/usr/local/bin/zabbix_sender")
+#         zabbix_sender => "/usr/local/bin/zabbix_sender"
+#       }
+#     }
+class LogStash::Outputs::Zabbix < LogStash::Outputs::Base
+ 
+  config_name "zabbix"
+  milestone 2
+
+  config :host, :validate => :string, :default => "localhost"
+  config :port, :validate => :number, :default => 10051
+  config :zabbix_sender, :validate => :path, :default => "/usr/local/bin/zabbix_sender"
+ 
+  public
+  def register
+    # nothing to do
+  end # def register
+ 
+  public
+  def receive(event)
+    return unless output?(event)
+ 
+    if !File.exists?(@zabbix_sender)
+      @logger.warn("Skipping zabbix output; zabbix_sender file is missing",
+                   :zabbix_sender => @zabbix_sender, :missed_event => event)
+      return
+    end
+ 
+    host = event["zabbix_host"]
+    if !host
+      @logger.warn("Skipping zabbix output; zabbix_host field is missing",
+                   :missed_event => event)
+      return
+    end
+    host = host.first if host.is_a?(Array)
+ 
+    item = event["zabbix_item"]
+    if !item
+      @logger.warn("Skipping zabbix output; zabbix_item field is missing",
+                   :missed_event => event)
+      return
+    end
+    item = item.first if item.is_a?(Array)
+ 
+    zmsg = event["message"]
+    zmsg = zmsg.gsub("\n", "\\n")
+    zmsg = zmsg.gsub(/"/, "\\\"")
+ 
+    cmd = "#{@zabbix_sender} -z #{@host} -p #{@port} -s #{host} -k #{item} -o \"#{zmsg}\" 2>/dev/null >/dev/null"
+ 
+    @logger.debug("Running zabbix command", :command => cmd)
+    begin
+      # TODO(sissel): Update this to use IO.popen so we can capture the output and
+      # log it accordingly.
+      system(cmd)
+    rescue => e
+      @logger.warn("Skipping zabbix output; error calling zabbix_sender",
+                   :command => cmd, :missed_event => event,
+                   :exception => e, :backtrace => e.backtrace)
+    end
+  end # def receive
+end # class LogStash::Outputs::Zabbix

data/lib/logstash/outputs/zeromq.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# Write events to a 0MQ PUB socket.
+#
+# You need to have the 0mq 2.1.x library installed to be able to use
+# this output plugin.
+#
+# The default settings will create a publisher connecting to a subscriber
+# bound to tcp://127.0.0.1:2120
+#
+class LogStash::Outputs::ZeroMQ < LogStash::Outputs::Base
+
+  config_name "zeromq"
+  milestone 2
+
+  default :codec, "json"
+
+  # 0mq socket address to connect or bind.
+  # Please note that `inproc://` will not work with logstashi.
+  # For each we use a context per thread.
+  # By default, inputs bind/listen and outputs connect.
+  config :address, :validate => :array, :default => ["tcp://127.0.0.1:2120"]
+
+  # The default logstash topologies work as follows:
+  #
+  # * pushpull - inputs are pull, outputs are push
+  # * pubsub - inputs are subscribers, outputs are publishers
+  # * pair - inputs are clients, inputs are servers
+  #
+  # If the predefined topology flows don't work for you,
+  # you can change the 'mode' setting
+  # TODO (lusis) add req/rep MAYBE
+  # TODO (lusis) add router/dealer
+  config :topology, :validate => ["pushpull", "pubsub", "pair"], :required => true
+
+  # This is used for the 'pubsub' topology only.
+  # On inputs, this allows you to filter messages by topic.
+  # On outputs, this allows you to tag a message for routing.
+  # NOTE: ZeroMQ does subscriber-side filtering
+  # NOTE: Topic is evaluated with `event.sprintf` so macros are valid here.
+  config :topic, :validate => :string, :default => ""
+
+  # Server mode binds/listens. Client mode connects.
+  config :mode, :validate => ["server", "client"], :default => "client"
+
+  # This exposes zmq_setsockopt for advanced tuning.
+  # See http://api.zeromq.org/2-1:zmq-setsockopt for details.
+  #
+  # This is where you would set values like:
+  #
+  # * ZMQ::HWM - high water mark
+  # * ZMQ::IDENTITY - named queues
+  # * ZMQ::SWAP_SIZE - space for disk overflow
+  #
+  # Example: sockopt => ["ZMQ::HWM", 50, "ZMQ::IDENTITY", "my_named_queue"]
+  config :sockopt, :validate => :hash
+
+  public
+  def register
+    require "ffi-rzmq"
+    require "logstash/util/zeromq"
+    self.class.send(:include, LogStash::Util::ZeroMQ)
+
+    if @mode == "server"
+      workers_not_supported("With 'mode => server', only one zeromq socket may bind to a port and may not be shared among threads. Going to single-worker mode for this plugin!")
+    end
+
+    # Translate topology shorthand to socket types
+    case @topology
+    when "pair"
+      zmq_const = ZMQ::PAIR
+    when "pushpull"
+      zmq_const = ZMQ::PUSH
+    when "pubsub"
+      zmq_const = ZMQ::PUB
+    end # case socket_type
+
+    @zsocket = context.socket(zmq_const)
+
+    error_check(@zsocket.setsockopt(ZMQ::LINGER, 1),
+                "while setting ZMQ::LINGER == 1)")
+
+    if @sockopt
+      setopts(@zsocket, @sockopt)
+    end
+
+    @address.each do |addr|
+      setup(@zsocket, addr)
+    end
+
+    @codec.on_event(&method(:publish))
+  end # def register
+
+  public
+  def teardown
+    error_check(@zsocket.close, "while closing the socket")
+  end # def teardown
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    @codec.encode(event)
+  end # def receive
+
+  def publish(payload)
+    @logger.debug? && @logger.debug("0mq: sending", :event => payload)
+    if @topology == "pubsub"
+      # TODO(sissel): Need to figure out how to fit this into the codecs system.
+      #@logger.debug("0mq output: setting topic to: #{event.sprintf(@topic)}")
+      #error_check(@zsocket.send_string(event.sprintf(@topic), ZMQ::SNDMORE),
+                  #"in topic send_string")
+    end
+    error_check(@zsocket.send_string(payload), "in send_string")
+  rescue => e
+    @logger.warn("0mq output exception", :address => @address, :exception => e)
+  end
+end # class LogStash::Outputs::ZeroMQ

data/lib/logstash/pipeline.rb

@@ -0,0 +1,286 @@
+# encoding: utf-8
+require "logstash/config/file"
+require "logstash/namespace"
+require "thread" # stdlib
+require "logstash/filters/base"
+require "logstash/inputs/base"
+require "logstash/outputs/base"
+require "logstash/errors"
+require "stud/interval" # gem stud
+
+class LogStash::Pipeline
+  def initialize(configstr)
+    @logger = Cabin::Channel.get(LogStash)
+    grammar = LogStashConfigParser.new
+    @config = grammar.parse(configstr)
+    if @config.nil?
+      raise LogStash::ConfigurationError, grammar.failure_reason
+    end
+
+    # This will compile the config to ruby and evaluate the resulting code.
+    # The code will initialize all the plugins and define the
+    # filter and output methods.
+    code = @config.compile
+    # The config code is hard to represent as a log message...
+    # So just print it.
+    @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
+    begin
+      eval(code)
+    rescue => e
+      raise
+    end
+
+    @input_to_filter = SizedQueue.new(20)
+
+    # If no filters, pipe inputs directly to outputs
+    if !filters?
+      @filter_to_output = @input_to_filter
+    else
+      @filter_to_output = SizedQueue.new(20)
+    end
+    @settings = {
+      "filter-workers" => 1,
+    }
+  end # def initialize
+
+  def ready?
+    return @ready
+  end
+
+  def started?
+    return @started
+  end
+
+  def configure(setting, value)
+    @settings[setting] = value
+  end
+
+  def filters?
+    return @filters.any?
+  end
+
+  def run
+    @started = true
+    @input_threads = []
+    start_inputs
+    start_filters if filters?
+    start_outputs
+
+    @ready = true
+
+    @logger.info("Pipeline started")
+    wait_inputs
+
+    # In theory there's nothing to do to filters to tell them to shutdown?
+    if filters?
+      shutdown_filters
+      wait_filters
+    end
+    shutdown_outputs
+    wait_outputs
+
+    @logger.info("Pipeline shutdown complete.")
+
+    # exit code
+    return 0
+  end # def run
+
+  def wait_inputs
+    @input_threads.each(&:join)
+  rescue Interrupt
+    # rbx does weird things during do SIGINT that I haven't debugged
+    # so we catch Interrupt here and signal a shutdown. For some reason the
+    # signal handler isn't invoked it seems? I dunno, haven't looked much into
+    # it.
+    shutdown
+  end
+
+  def shutdown_filters
+    @input_to_filter.push(LogStash::ShutdownSignal)
+  end
+
+  def wait_filters
+    @filter_threads.each(&:join) if @filter_threads
+  end
+
+  def shutdown_outputs
+    # nothing, filters will do this
+    @filter_to_output.push(LogStash::ShutdownSignal)
+  end
+
+  def wait_outputs
+    # Wait for the outputs to stop
+    @output_threads.each(&:join)
+  end
+
+  def start_inputs
+    moreinputs = []
+    @inputs.each do |input|
+      if input.threadable && input.threads > 1
+        (input.threads-1).times do |i|
+          moreinputs << input.clone
+        end
+      end
+    end
+    @inputs += moreinputs
+
+    @inputs.each do |input|
+      input.register
+      start_input(input)
+    end
+  end
+
+  def start_filters
+    @filters.each(&:register)
+    @filter_threads = @settings["filter-workers"].times.collect do
+      Thread.new { filterworker }
+    end
+
+    # Set up the periodic flusher thread.
+    @flusher_thread = Thread.new { Stud.interval(5) { filter_flusher } }
+  end
+
+  def start_outputs
+    @output_threads = [
+      Thread.new { outputworker }
+    ]
+  end
+
+  def start_input(plugin)
+    @input_threads << Thread.new { inputworker(plugin) }
+  end
+
+  def inputworker(plugin)
+    LogStash::Util::set_thread_name("<#{plugin.class.config_name}")
+    begin
+      plugin.run(@input_to_filter)
+    rescue LogStash::ShutdownSignal
+      return
+    rescue => e
+      if @logger.debug?
+        @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
+                             :plugin => plugin.inspect, :error => e.to_s,
+                             :exception => e.class,
+                             :stacktrace => e.backtrace.join("\n")))
+      else
+        @logger.error(I18n.t("logstash.pipeline.worker-error",
+                             :plugin => plugin.inspect, :error => e))
+      end
+      puts e.backtrace if @logger.debug?
+      plugin.teardown
+      sleep 1
+      retry
+    end
+  rescue LogStash::ShutdownSignal
+    # nothing
+  ensure
+    plugin.teardown
+  end # def inputworker
+
+  def filterworker
+    LogStash::Util::set_thread_name("|worker")
+    begin
+      while true
+        event = @input_to_filter.pop
+        if event == LogStash::ShutdownSignal
+          @input_to_filter.push(event)
+          break
+        end
+
+
+        # TODO(sissel): we can avoid the extra array creation here
+        # if we don't guarantee ordering of origin vs created events.
+        # - origin event is one that comes in naturally to the filter worker.
+        # - created events are emitted by filters like split or metrics
+        events = [event]
+        filter(event) do |newevent|
+          events << newevent
+        end
+        events.each do |event|
+          next if event.cancelled?
+          @filter_to_output.push(event)
+        end
+      end
+    rescue => e
+      @logger.error("Exception in filterworker", "exception" => e, "backtrace" => e.backtrace)
+    end
+
+    @filters.each(&:teardown)
+  end # def filterworker
+
+  def outputworker
+    LogStash::Util::set_thread_name(">output")
+    @outputs.each(&:register)
+    @outputs.each(&:worker_setup)
+    while true
+      event = @filter_to_output.pop
+      break if event == LogStash::ShutdownSignal
+      output(event)
+    end # while true
+    @outputs.each(&:teardown)
+  end # def outputworker
+
+  # Shutdown this pipeline.
+  #
+  # This method is intended to be called from another thread
+  def shutdown
+    @input_threads.each do |thread|
+      # Interrupt all inputs
+      @logger.info("Sending shutdown signal to input thread",
+                   :thread => thread)
+      thread.raise(LogStash::ShutdownSignal)
+      begin
+        thread.wakeup # in case it's in blocked IO or sleeping
+      rescue ThreadError
+      end
+
+      # Sometimes an input is stuck in a blocking I/O
+      # so we need to tell it to teardown directly
+      @inputs.each do |input|
+        input.teardown
+      end
+    end
+
+    # No need to send the ShutdownSignal to the filters/outputs nor to wait for
+    # the inputs to finish, because in the #run method we wait for that anyway.
+  end # def shutdown
+
+  def plugin(plugin_type, name, *args)
+    args << {} if args.empty?
+    klass = LogStash::Plugin.lookup(plugin_type, name)
+    return klass.new(*args)
+  end
+
+  def filter(event, &block)
+    @filter_func.call(event, &block)
+  end
+
+  def output(event)
+    @output_func.call(event)
+  end
+
+  def filter_flusher
+    events = []
+    @filters.each do |filter|
+
+      # Filter any events generated so far in this flush.
+      events.each do |event|
+        # TODO(sissel): watchdog on flush filtration?
+        unless event.cancelled?
+          filter.filter(event)
+        end
+      end
+
+      # TODO(sissel): watchdog on flushes?
+      if filter.respond_to?(:flush)
+        flushed = filter.flush
+        events += flushed if !flushed.nil? && flushed.any?
+      end
+    end
+
+    events.each do |event|
+      @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
+      @filter_to_output.push(event) unless event.cancelled?
+    end
+  end # def filter_flusher
+end # class Pipeline