logstash-lib 1.3.2

data/lib/logstash/outputs/lumberjack.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+class LogStash::Outputs::Lumberjack < LogStash::Outputs::Base
+
+  config_name "lumberjack"
+  milestone 1
+
+  # list of addresses lumberjack can send to
+  config :hosts, :validate => :array, :required => true
+
+  # the port to connect to
+  config :port, :validate => :number, :required => true
+
+  # ssl certificate to use
+  config :ssl_certificate, :validate => :path, :required => true
+
+  # window size
+  config :window_size, :validate => :number, :default => 5000
+
+  public
+  def register
+    require 'lumberjack/client'
+    connect
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    begin
+      @client.write(event.to_hash)
+    rescue Exception => e
+      @logger.error("Client write error", :e => e, :backtrace => e.backtrace)
+      connect
+      retry
+    end
+  end # def receive
+
+  private 
+  def connect
+    @logger.info("Connecting to lumberjack server.", :addresses => @hosts, :port => @port, 
+        :ssl_certificate => @ssl_certificate, :window_size => @window_size)
+    begin
+      @client = Lumberjack::Client.new(:addresses => @hosts, :port => @port, 
+        :ssl_certificate => @ssl_certificate, :window_size => @window_size)
+    rescue Exception => e
+      @logger.error("All hosts unavailable, sleeping", :hosts => @hosts, :e => e, 
+        :backtrace => e.backtrace)
+      sleep(10)
+      retry
+    end
+  end
+end

data/lib/logstash/outputs/metriccatcher.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "json"
+
+# This output ships metrics to MetricCatcher, allowing you to
+# utilize Coda Hale's Metrics.
+#
+# More info on MetricCatcher: https://github.com/clearspring/MetricCatcher
+#
+# At Clearspring, we use it to count the response codes from Apache logs:
+#     metriccatcher {
+#         host => "localhost"
+#         port => "1420"
+#         type => "apache-access"
+#         fields => [ "response" ]
+#         meter => [ "%{host}.apache.response.%{response}", "1" ]
+#     }
+class LogStash::Outputs::MetricCatcher < LogStash::Outputs::Base
+  config_name "metriccatcher"
+  milestone 2
+
+  # The address of the MetricCatcher
+  config :host, :validate => :string, :default => "localhost"
+  # The port to connect on your MetricCatcher
+  config :port, :validate => :number, :default => 1420
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value.
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :gauge, :validate => :hash
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value. Example:
+  #
+  #   counter => [ "%{host}.apache.hits.%{response}, "1" ]
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :counter, :validate => :hash
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value.
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :meter, :validate => :hash
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value.
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :biased, :validate => :hash
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value.
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :uniform, :validate => :hash
+
+  # The metrics to send. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key
+  # of the metric name, value of the metric value. Example:
+  #
+  #   timer => [ "%{host}.apache.response_time, "%{response_time}" ]
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :timer, :validate => :hash
+
+  def register
+    @socket = UDPSocket.new
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    @@metric_types.each do |metric_type|
+      if instance_variable_defined?("@#{metric_type}")
+        instance_variable_get("@#{metric_type}").each do |metric_name, metric_value|
+          message = [{
+            "name"      => event.sprintf(metric_name),
+            "type"      => event.sprintf(metric_type),
+            "value"     => event.sprintf(metric_value).to_f,
+            "timestamp" => event.sprintf("%{+%s}.") + Time.now.usec.to_s
+          }]
+
+          @socket.send(message.to_json, 0, @host, @port)
+        end # instance_variable_get("@#{metric_type}").each_slice
+      end # if
+    end # @metric_types.each
+  end # def receive
+end # class LogStash::Outputs::MetricCatcher

data/lib/logstash/outputs/mongodb.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
+
+  config_name "mongodb"
+  milestone 2
+
+  # a MongoDB URI to connect to
+  # See http://docs.mongodb.org/manual/reference/connection-string/
+  config :uri, :validate => :string, :required => true
+  
+  # The database to use
+  config :database, :validate => :string, :required => true
+   
+  # The collection to use. This value can use %{foo} values to dynamically
+  # select a collection based on data in the event.
+  config :collection, :validate => :string, :required => true
+
+  # If true, store the @timestamp field in mongodb as an ISODate type instead
+  # of an ISO8601 string.  For more information about this, see
+  # http://www.mongodb.org/display/DOCS/Dates
+  config :isodate, :validate => :boolean, :default => false
+
+  # Number of seconds to wait after failure before retrying
+  config :retry_delay, :validate => :number, :default => 3, :required => false
+
+  # If true, a _id field will be added to the document before insertion.
+  # The _id field will use the timestamp of the event and overwrite an existing
+  # _id field in the event.
+  config :generateId, :validate => :boolean, :default => false
+
+  public
+  def register
+    require "mongo"
+    uriParsed=Mongo::URIParser.new(@uri)
+    conn = uriParsed.connection({})
+    if uriParsed.auths.length > 0
+      uriParsed.auths.each do |auth|
+        if !auth['db_name'].nil?
+          conn.add_auth(auth['db_name'], auth['username'], auth['password'], nil)
+        end 
+      end
+      conn.apply_saved_authentication()
+    end
+    @db = conn.db(@database)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    begin
+      if @isodate
+        # the mongodb driver wants time values as a ruby Time object.
+        # set the @timestamp value of the document to a ruby Time object, then.
+        document = event.to_hash
+      else
+        document = event.to_hash.merge("@timestamp" => event["@timestamp"].to_json)
+      end
+      if @generateId
+        document['_id'] = BSON::ObjectId.new(nil, event["@timestamp"])
+      end
+      @db.collection(event.sprintf(@collection)).insert(document)
+    rescue => e
+      @logger.warn("Failed to send event to MongoDB", :event => event, :exception => e,
+                   :backtrace => e.backtrace)
+      if e.error_code == 11000
+          # On a duplicate key error, skip the insert.
+          # We could check if the duplicate key err is the _id key
+          # and generate a new primary key.
+          # If the duplicate key error is on another field, we have no way
+          # to fix the issue.
+      else
+        sleep @retry_delay
+        retry
+      end
+    end
+  end # def receive
+end # class LogStash::Outputs::Mongodb

data/lib/logstash/outputs/nagios.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+
+# The nagios output is used for sending passive check results to nagios via the
+# nagios command file. 
+#
+# For this output to work, your event must have the following fields:
+#
+#  * "nagios_host"
+#  * "nagios_service"
+#
+# These fields are supported, but optional:
+#
+#  * "nagios_annotation"
+#  * "nagios_level"
+#
+# There are two configuration options:
+#
+#  * commandfile - The location of the Nagios external command file
+#  * nagios_level - Specifies the level of the check to be sent. Defaults to
+#    CRITICAL and can be overriden by setting the "nagios_level" field to one
+#    of "OK", "WARNING", "CRITICAL", or "UNKNOWN" 
+#
+#         match => [ "message", "(error|ERROR|CRITICAL)" ]
+#
+#     output{
+#       if [message] =~ /(error|ERROR|CRITICAL)/ {
+#         nagios {
+#           # your config here
+#         }
+#       }
+#     }
+class LogStash::Outputs::Nagios < LogStash::Outputs::Base
+
+  config_name "nagios"
+  milestone 2
+
+  # The path to your nagios command file
+  config :commandfile, :validate => :path, :default => "/var/lib/nagios3/rw/nagios.cmd"
+
+  # The Nagios check level. Should be one of 0=OK, 1=WARNING, 2=CRITICAL,
+  # 3=UNKNOWN. Defaults to 2 - CRITICAL.
+  config :nagios_level, :validate => [ "0", "1", "2", "3" ], :default => "2"
+
+  public
+  def register
+    # nothing to do
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    if !File.exists?(@commandfile)
+      @logger.warn("Skipping nagios output; command file is missing",
+                   :commandfile => @commandfile, :missed_event => event)
+      return
+    end
+
+    # TODO(petef): if nagios_host/nagios_service both have more than one
+    # value, send multiple alerts. They will have to match up together by
+    # array indexes (host/service combos) and the arrays must be the same
+    # length.
+
+    host = event["nagios_host"]
+    if !host
+      @logger.warn("Skipping nagios output; nagios_host field is missing",
+                   :missed_event => event)
+      return
+    end
+
+    service = event["nagios_service"]
+    if !service
+      @logger.warn("Skipping nagios output; nagios_service field is missing",
+                   "missed_event" => event)
+      return
+    end
+
+    annotation = event["nagios_annotation"]
+    level = @nagios_level
+
+    if event["nagios_level"]
+      event_level = [*event["nagios_level"]]
+      case event_level[0].downcase
+      when "ok"
+        level = "0"
+      when "warning"
+        level = "1"
+      when "critical"
+        level = "2"
+      when "unknown"
+        level = "3"
+      else
+        @logger.warn("Invalid Nagios level. Defaulting to CRITICAL", :data => event_level)
+      end
+    end
+
+    cmd = "[#{Time.now.to_i}] PROCESS_SERVICE_CHECK_RESULT;#{host};#{service};#{level};"
+    if annotation
+      cmd += "#{annotation}: "
+    end
+    # In the multi-line case, escape the newlines for the nagios command file
+    cmd += (event["message"] || "<no message>").gsub("\n", "\\n")
+
+    @logger.debug("Opening nagios command file", :commandfile => @commandfile,
+                  :nagios_command => cmd)
+    begin
+      File.open(@commandfile, "r+") do |f|
+        f.puts(cmd)
+        f.flush # TODO(sissel): probably don't need this.
+      end
+    rescue => e
+      @logger.warn("Skipping nagios output; error writing to command file",
+                   :commandfile => @commandfile, :missed_event => event,
+                   :exception => e, :backtrace => e.backtrace)
+    end
+  end # def receive
+end # class LogStash::Outputs::Nagios

data/lib/logstash/outputs/nagios_nsca.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# The nagios_nsca output is used for sending passive check results to Nagios
+# through the NSCA protocol.
+#
+# This is useful if your Nagios server is not the same as the source host from
+# where you want to send logs or alerts. If you only have one server, this
+# output is probably overkill # for you, take a look at the 'nagios' output
+# instead.
+#
+# Here is a sample config using the nagios_nsca output:
+#     output {
+#       nagios_nsca {
+#         # specify the hostname or ip of your nagios server
+#         host => "nagios.example.com"
+#
+#         # specify the port to connect to
+#         port => 5667
+#       }
+#     }
+
+class LogStash::Outputs::NagiosNsca < LogStash::Outputs::Base
+
+  config_name "nagios_nsca"
+  milestone 1
+
+  # The status to send to nagios. Should be 0 = OK, 1 = WARNING, 2 = CRITICAL, 3 = UNKNOWN
+  config :nagios_status, :validate => :string, :required => true
+
+  # The nagios host or IP to send logs to. It should have a NSCA daemon running.
+  config :host, :validate => :string, :default => "localhost"
+
+  # The port where the NSCA daemon on the nagios host listens.
+  config :port, :validate => :number, :default => 5667
+
+  # The path to the 'send_nsca' binary on the local host.
+  config :send_nsca_bin, :validate => :path, :default => "/usr/sbin/send_nsca"
+
+  # The path to the send_nsca config file on the local host.
+  # Leave blank if you don't want to provide a config file.
+  config :send_nsca_config, :validate => :path
+
+  # The nagios 'host' you want to submit a passive check result to. This
+  # parameter accepts interpolation, e.g. you can use @source_host or other
+  # logstash internal variables.
+  config :nagios_host, :validate => :string, :default => "%{host}"
+
+  # The nagios 'service' you want to submit a passive check result to. This
+  # parameter accepts interpolation, e.g. you can use @source_host or other
+  # logstash internal variables.
+  config :nagios_service, :validate => :string, :default => "LOGSTASH"
+
+  # The format to use when writing events to nagios. This value
+  # supports any string and can include %{name} and other dynamic
+  # strings.
+  config :message_format, :validate => :string, :default => "%{@timestamp} %{host}: %{message}"
+
+  public
+  def register
+    #nothing for now
+  end
+
+  public
+  def receive(event)
+    # exit if type or tags don't match
+    return unless output?(event)
+
+    # catch logstash shutdown
+    if event == LogStash::SHUTDOWN
+      finished
+      return
+    end
+
+    # skip if 'send_nsca' binary doesn't exist
+    if !File.exists?(@send_nsca_bin)
+      @logger.warn("Skipping nagios_nsca output; send_nsca_bin file is missing",
+                   "send_nsca_bin" => @send_nsca_bin, "missed_event" => event)
+      return
+    end
+
+    # interpolate params
+    nagios_host = event.sprintf(@nagios_host)
+    nagios_service = event.sprintf(@nagios_service)
+
+    # escape basic things in the log message
+    # TODO: find a way to escape the message correctly
+    msg = event.sprintf(@message_format)
+    msg.gsub!("\n", "<br/>")
+    msg.gsub!("'", "&#146;")
+
+    status = event.sprintf(@nagios_status)
+    if status.to_i.to_s != status # Check it round-trips to int correctly
+      msg = "status '#{status}' is not numeric"
+      status = 2
+    else
+      status = status.to_i
+      if status > 3 || status < 0
+         msg "status must be > 0 and <= 3, not #{status}"
+         status = 2
+      end
+    end
+
+    # build the command
+    # syntax: echo '<server>!<nagios_service>!<status>!<text>'  | \
+    #           /usr/sbin/send_nsca -H <nagios_host> -d '!' -c <nsca_config>"
+    cmd = %(echo '#{nagios_host}~#{nagios_service}~#{status}~#{msg}' |)
+    cmd << %( #{@send_nsca_bin} -H #{@host} -p #{@port} -d '~')
+    cmd << %( -c #{@send_nsca_config}) if @send_nsca_config
+    cmd << %( 2>/dev/null >/dev/null)
+    @logger.debug("Running send_nsca command", "nagios_nsca_command" => cmd)
+
+    begin
+      system cmd
+    rescue => e
+      @logger.warn("Skipping nagios_nsca output; error calling send_nsca",
+                   "error" => $!, "nagios_nsca_command" => cmd,
+                   "missed_event" => event)
+      @logger.debug("Backtrace", e.backtrace)
+    end
+  end # def receive
+end # class LogStash::Outputs::NagiosNsca