logstash-lib 1.3.2

data/lib/logstash/outputs/null.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# A null output. This is useful for testing logstash inputs and filters for
+# performance.
+class LogStash::Outputs::Null < LogStash::Outputs::Base
+  config_name "null"
+  milestone 3
+
+  public
+  def register
+  end # def register
+
+  public
+  def receive(event)
+  end # def event
+end # class LogStash::Outputs::Null

data/lib/logstash/outputs/opentsdb.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "socket"
+
+# This output allows you to pull metrics from your logs and ship them to
+# opentsdb. Opentsdb is an open source tool for storing and graphing metrics.
+#
+class LogStash::Outputs::Opentsdb < LogStash::Outputs::Base
+  config_name "opentsdb"
+  milestone 1
+
+  # Enable debugging. Tries to pretty-print the entire event object.
+  config :debug, :validate => :boolean
+
+  # The address of the opentsdb server.
+  config :host, :validate => :string, :default => "localhost"
+
+  # The port to connect on your graphite server.
+  config :port, :validate => :number, :default => 4242
+
+  # The metric(s) to use. This supports dynamic strings like %{source_host}
+  # for metric names and also for values. This is an array field with key
+  # of the metric name, value of the metric value, and multiple tag,values . Example:
+  #
+  #     [
+  #       "%{host}/uptime",
+  #       %{uptime_1m} " ,
+  #       "hostname" ,
+  #       "%{host}
+  #       "anotherhostname" ,
+  #       "%{host}
+  #     ]
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :metrics, :validate => :array, :required => true
+
+  def register
+    connect
+  end # def register
+
+  def connect
+    # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory.
+    begin
+      @socket = TCPSocket.new(@host, @port)
+    rescue Errno::ECONNREFUSED => e
+      @logger.warn("Connection refused to opentsdb server, sleeping...",
+                   :host => @host, :port => @port)
+      sleep(2)
+      retry
+    end
+  end # def connect
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    # Opentsdb message format: put metric timestamp value tagname=tagvalue tag2=value2\n
+
+    # Catch exceptions like ECONNRESET and friends, reconnect on failure.
+    begin
+      name = metrics[0]
+      value = metrics[1]
+      tags = metrics[2..-1]
+
+      # The first part of the message
+      message = ['put',
+                 event.sprintf(name),
+                 event.sprintf("%{+%s}"),
+                 event.sprintf(value),
+      ].join(" ")
+
+      # If we have have tags we need to add it to the message
+      event_tags = []
+      unless tags.nil?
+        Hash[*tags.flatten].each do |tag_name,tag_value|
+          # Interprete variables if neccesary
+          real_tag_name = event.sprintf(tag_name)
+          real_tag_value =  event.sprintf(tag_value)
+          event_tags << [real_tag_name , real_tag_value ].join('=')
+        end
+        message+=' '+event_tags.join(' ')
+      end
+
+      # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory.
+      begin
+        @socket.puts(message)
+      rescue Errno::EPIPE, Errno::ECONNRESET => e
+        @logger.warn("Connection to opentsdb server died",
+                     :exception => e, :host => @host, :port => @port)
+        sleep(2)
+        connect
+      end
+
+      # TODO(sissel): resend on failure
+      # TODO(sissel): Make 'resend on failure' tunable; sometimes it's OK to
+      # drop metrics.
+    end # @metrics.each
+  end # def receive
+end # class LogStash::Outputs::Opentsdb

data/lib/logstash/outputs/pagerduty.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# PagerDuty output
+# Send specific events to PagerDuty for alerting
+
+class LogStash::Outputs::PagerDuty < LogStash::Outputs::Base
+  config_name "pagerduty"
+  milestone 1
+
+  # Service API Key
+  config :service_key, :validate => :string, :required => true
+
+  # The service key to use
+  # You'll need to set this up in PD beforehand
+  config :incident_key, :validate => :string, :default => "logstash/%{host}/%{type}"
+
+  # Event type
+  config :event_type, :validate => ["trigger", "acknowledge", "resolve"], :default => "trigger"
+
+  # Custom description
+  config :description, :validate => :string, :default => "Logstash event for %{host}"
+
+  # Event details
+  # These might be keys from the logstash event you wish to include
+  # tags are automatically included if detected so no need to add them here
+  config :details, :validate => :hash, :default => {"timestamp" => "%{@timestamp}", "message" => "%{message}"}
+
+  # PagerDuty API url
+  # You shouldn't need to change this
+  # This allows for flexibility
+  # should PD iterate the API
+  # and Logstash hasn't updated yet
+  config :pdurl, :validate => :string, :default => "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+
+  public
+  def register
+    require 'net/https'
+    require 'uri'
+    @pd_uri = URI.parse(@pdurl)
+    @client = Net::HTTP.new(@pd_uri.host, @pd_uri.port)
+    if @pd_uri.scheme == "https"
+      @client.use_ssl = true
+      #@client.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # PagerDuty cert doesn't verify oob
+      @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+   
+    pd_event = Hash.new
+    pd_event[:service_key] = "#{@service_key}"
+    pd_event[:incident_key] = event.sprintf(@incident_key)
+    pd_event[:event_type] = "#{@event_type}"
+    pd_event[:description] = event.sprintf(@description)
+    pd_event[:details] = Hash.new
+    @details.each do |key, value|
+      @logger.debug("PD Details added:" , key => event.sprintf(value))
+      pd_event[:details]["#{key}"] = event.sprintf(value)
+    end
+    pd_event[:details][:tags] = @tags if @tags
+
+    @logger.info("PD Event", :event => pd_event)
+    begin
+      request = Net::HTTP::Post.new(@pd_uri.path)
+      request.body = pd_event.to_json
+      @logger.debug("PD Request", :request => request.inspect)
+      response = @client.request(request)
+      @logger.debug("PD Response", :response => response.body)
+
+    rescue Exception => e
+      @logger.debug("PD Unhandled exception", :pd_error => e.backtrace)
+    end
+  end # def receive
+end # class LogStash::Outputs::PagerDuty

data/lib/logstash/outputs/pipe.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+
+# Pipe output.
+#
+# Pipe events to stdin of another program. You can use fields from the
+# event as parts of the command.
+# WARNING: This feature can cause logstash to fork off multiple children if you are not carefull with per-event commandline.
+class LogStash::Outputs::Pipe < LogStash::Outputs::Base
+
+  config_name "pipe"
+  milestone 1
+
+  # The format to use when writing events to the pipe. This value
+  # supports any string and can include %{name} and other dynamic
+  # strings.
+  #
+  # If this setting is omitted, the full json representation of the
+  # event will be written as a single line.
+  config :message_format, :validate => :string
+
+  # Command line to launch and pipe to
+  config :command, :validate => :string, :required => true
+
+  # Close pipe that hasn't been used for TTL seconds. -1 or 0 means never close.
+  config :ttl, :validate => :number, :default => 10
+  public
+  def register
+    @pipes = {}
+    @last_stale_cleanup_cycle = Time.now
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    command = event.sprintf(@command)
+    pipe = get_pipe(command)
+
+    if @message_format
+      output = event.sprintf(@message_format) + "\n"
+    else
+      output = event.to_json
+    end
+
+    begin
+      pipe.puts(output)
+    rescue IOError, Errno::EPIPE => e
+      @logger.error("Error writing to pipe, closing pipe.", :command => command, :pipe => pipe)
+      drop_pipe(command)
+    end
+
+    close_stale_pipes
+  end # def receive
+
+  def teardown
+    @logger.info("Teardown: closing pipes")
+    @pipes.each do |command, pipe|
+      begin
+        drop_pipe(command)
+        @logger.debug("Closed pipe #{command}", :pipe => pipe)
+      rescue Exception => e
+        @logger.error("Excpetion while closing pipes.", :exception => e)
+      end
+    end
+    finished
+  end
+
+  private
+  # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
+  def close_stale_pipes
+    return if @ttl <= 0
+    now = Time.now
+    return unless now - @last_stale_cleanup_cycle >= @ttl
+    @logger.info("Starting stale pipes cleanup cycle", :pipes => @pipes)
+    inactive_pipes = @pipes.select { |command, pipe| not pipe.active }
+    @logger.debug("%d stale pipes found" % inactive_pipes.count, :inactive_pipes => inactive_pipes)
+    inactive_pipes.each do |command, pipe|
+      drop_pipe(command)
+    end
+    # mark all pipes as inactive, a call to write will mark them as active again
+    @pipes.each { |command, pipe| pipe.active = false }
+    @last_stale_cleanup_cycle = now
+  end
+
+  def drop_pipe(command)
+      return unless @pipes.include? command
+      @logger.info("Closing pipe \"%s\"" % command)
+      begin
+        @pipes[command].close
+      rescue Exception => e
+        @logger.warn("Failed to close pipe.", :error => e, :command => command)
+      end
+      @pipes.delete(command)
+  end
+
+  def get_pipe(command)
+    return @pipes[command] if @pipes.include?(command)
+
+    @logger.info("Opening pipe", :command => command)
+
+    @pipes[command] = PipeWrapper.new(command, mode="a+")
+  end
+end # class LogStash::Outputs::Pipe
+
+class PipeWrapper
+  attr_accessor :active
+  def initialize(command, mode="a+")
+    @pipe = IO.popen(command, mode)
+    @active = false
+  end
+
+  def method_missing?(m, *args)
+    if @pipe.respond_to? m
+      @pipe.send(m, *args)
+    else
+      raise NoMethodError
+    end
+  end
+
+  def puts(txt)
+    @pipe.puts(txt)
+    @pipe.flush
+    @active = true
+  end
+
+  def write(txt)
+    @pipe.write(txt)
+    @active = true
+  end
+end

data/lib/logstash/outputs/rabbitmq.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
+# or later version (3.x is recommended).
+#
+# Relevant links:
+#
+# * RabbitMQ: <http://www.rabbitmq.com/>
+# * March Hare: <http://rubymarchhare.info>
+# * Bunny: <http://rubybunny.info>
+class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
+  EXCHANGE_TYPES = ["fanout", "direct", "topic"]
+
+  config_name "rabbitmq"
+  milestone 1
+
+
+  #
+  # Connection
+  #
+
+  # RabbitMQ server address
+  config :host, :validate => :string, :required => true
+
+  # RabbitMQ port to connect on
+  config :port, :validate => :number, :default => 5672
+
+  # RabbitMQ username
+  config :user, :validate => :string, :default => "guest"
+
+  # RabbitMQ password
+  config :password, :validate => :password, :default => "guest"
+
+  # The vhost to use. If you don't know what this is, leave the default.
+  config :vhost, :validate => :string, :default => "/"
+
+  # Enable or disable SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # Validate SSL certificate
+  config :verify_ssl, :validate => :boolean, :default => false
+
+  # Enable or disable logging
+  config :debug, :validate => :boolean, :default => false
+
+
+
+  #
+  # Exchange
+  #
+
+
+  # The exchange type (fanout, topic, direct)
+  config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
+
+  # The name of the exchange
+  config :exchange, :validate => :string, :required => true
+
+  # Key to route to by default. Defaults to 'logstash'
+  #
+  # * Routing keys are ignored on fanout exchanges.
+  config :key, :validate => :string, :default => "logstash"
+
+  # Is this exchange durable? (aka; Should it survive a broker restart?)
+  config :durable, :validate => :boolean, :default => true
+
+  # Should RabbitMQ persist messages to disk?
+  config :persistent, :validate => :boolean, :default => true
+
+
+
+  def initialize(params)
+    params["codec"] = "json" if !params["codec"]
+
+    super
+  end
+
+  # Use MarchHare on JRuby to avoid IO#select CPU spikes
+  # (see github.com/ruby-amqp/bunny/issues/95).
+  #
+  # On MRI, use Bunny.
+  #
+  # See http://rubybunny.info and http://rubymarchhare.info
+  # for the docs.
+  if RUBY_ENGINE == "jruby"
+    require "logstash/outputs/rabbitmq/march_hare"
+
+    include MarchHareImpl
+  else
+    require "logstash/outputs/rabbitmq/bunny"
+
+    include BunnyImpl
+  end
+end # class LogStash::Outputs::RabbitMQ

data/lib/logstash/outputs/rabbitmq/bunny.rb

@@ -0,0 +1,135 @@
+# encoding: utf-8
+class LogStash::Outputs::RabbitMQ
+  module BunnyImpl
+
+    #
+    # API
+    #
+
+    def register
+      require "bunny"
+
+      @logger.info("Registering output", :plugin => self)
+
+      connect
+      declare_exchange
+    end # def register
+
+
+    def receive(event)
+      return unless output?(event)
+
+      @logger.debug("Sending event", :destination => to_s, :event => event, :key => key)
+      key = event.sprintf(@key) if @key
+
+      begin
+        publish_serialized(event.to_json, key)
+      rescue JSON::GeneratorError => e
+        @logger.warn("Trouble converting event to JSON", :exception => e,
+                     :event => event)
+      end
+    end
+
+    def publish_serialized(message, key = @key)
+      begin
+        if @x
+          @x.publish(message, :persistent => @persistent, :routing_key => key)
+        else
+          @logger.warn("Tried to send a message, but not connected to RabbitMQ yet.")
+        end
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        connect
+        declare_exchange
+        retry
+      end
+    end
+
+    def to_s
+      return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
+    end
+
+    def teardown
+      @conn.close if @conn && @conn.open?
+      @conn = nil
+
+      finished
+    end
+
+
+
+    #
+    # Implementation
+    #
+
+    def connect
+      @vhost       ||= Bunny::DEFAULT_HOST
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= AMQ::Protocol::DEFAULT_PORT
+      @routing_key ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :automatically_recover => false
+      }
+      @settings[:user]      = @user || Bunny::DEFAULT_USER
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                Bunny::DEFAULT_PASSWORD
+                              end
+
+      @settings[:log_level] = if @debug
+                                :debug
+                              else
+                                :error
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      @settings[:verify_ssl] = @verify_ssl if @verify_ssl
+
+      proto                  = if @ssl
+                                 "amqp"
+                               else
+                                 "amqps"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      begin
+        @conn = Bunny.new(@settings)
+
+        @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+        return if terminating?
+        @conn.start
+
+        @ch = @conn.create_channel
+        @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        retry
+      end
+    end
+
+    def declare_exchange
+      @logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
+                    :durable => @durable)
+      @x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
+    end
+  end # BunnyImpl
+end # LogStash::Outputs::RabbitMQ