logstash-lib 1.3.2

data/lib/logstash/outputs/email.rb

@@ -0,0 +1,299 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# Send email when any event is received.
+class LogStash::Outputs::Email < LogStash::Outputs::Base
+
+  config_name "email"
+  milestone 1
+
+  # This setting is deprecated in favor of logstash's "conditionals" feature
+  # If you were using this setting previously, please use conditionals instead.
+  #
+  # If you need help converting your older 'match' setting to a conditional,
+  # I welcome you to join the #logstash irc channel on freenode or to email
+  # the logstash-users@googlegroups.com mailling list and ask for help! :)
+  config :match, :validate => :hash, :deprecated => true
+
+  # Who to send this email to?
+  # A fully qualified email address to send to
+  #
+  # This field also accept a comma separated list of emails like 
+  # "me@host.com, you@host.com"
+  #
+  # You can also use dynamic field from the event with the %{fieldname} syntax.
+  config :to, :validate => :string, :required => true
+
+  # The From setting for email - fully qualified email address for the From:
+  config :from, :validate => :string, :default => "logstash.alert@nowhere.com"
+
+  # The Reply-To setting for email - fully qualified email address is required
+  # here.
+  config :replyto, :validate => :string
+
+  # Who to CC on this email?
+  #
+  # See "to" setting for what is valid here.
+  config :cc, :validate => :string
+
+  # how to send email: either smtp or sendmail - default to 'smtp'
+  config :via, :validate => :string, :default => "smtp"
+
+  # the options to use:
+  # smtp: address, port, enable_starttls_auto, user_name, password, authentication(bool), domain
+  # sendmail: location, arguments
+  # If you do not specify anything, you will get the following equivalent code set in
+  # every new mail object:
+  #
+  #   Mail.defaults do
+  #     delivery_method :smtp, { :address              => "localhost",
+  #                              :port                 => 25,
+  #                              :domain               => 'localhost.localdomain',
+  #                              :user_name            => nil,
+  #                              :password             => nil,
+  #                              :authentication       => nil,(plain, login and cram_md5)
+  #                              :enable_starttls_auto => true  }
+  #
+  #     retriever_method :pop3, { :address             => "localhost",
+  #                               :port                => 995,
+  #                               :user_name           => nil,
+  #                               :password            => nil,
+  #                               :enable_ssl          => true }
+  #   end
+  #
+  #   Mail.delivery_method.new  #=> Mail::SMTP instance
+  #   Mail.retriever_method.new #=> Mail::POP3 instance
+  #
+  # Each mail object inherits the default set in Mail.delivery_method, however, on
+  # a per email basis, you can override the method:
+  #
+  #   mail.delivery_method :sendmail
+  #
+  # Or you can override the method and pass in settings:
+  #
+  #   mail.delivery_method :sendmail, { :address => 'some.host' }
+  #
+  # You can also just modify the settings:
+  #
+  #   mail.delivery_settings = { :address => 'some.host' }
+  #
+  # The passed in hash is just merged against the defaults with +merge!+ and the result
+  # assigned the mail object.  So the above example will change only the :address value
+  # of the global smtp_settings to be 'some.host', keeping all other values
+  config :options, :validate => :hash, :default => {}
+
+  # subject for email
+  config :subject, :validate => :string, :default => ""
+
+  # body for email - just plain text
+  config :body, :validate => :string, :default => ""
+
+  # body for email - can contain html markup
+  config :htmlbody, :validate => :string, :default => ""
+
+  # attachments - has of name of file and file location
+  config :attachments, :validate => :array, :default => []
+
+  # contenttype : for multipart messages, set the content type and/or charset of the html part
+  config :contenttype, :validate => :string, :default => "text/html; charset=UTF-8"
+
+  public
+  def register
+    require "mail"
+
+    # Mail uses instance_eval which changes the scope of self so @options is
+    # inaccessible from inside 'Mail.defaults'. So set a local variable instead.
+    options = @options
+
+    if @via == "smtp"
+      Mail.defaults do
+        delivery_method :smtp, {
+          :address              => options.fetch("smtpIporHost", "localhost"),
+          :port                 => options.fetch("port", 25),
+          :domain               => options.fetch("domain", "localhost"),
+          :user_name            => options.fetch("userName", nil),
+          :password             => options.fetch("password", nil),
+          :authentication       => options.fetch("authenticationType", nil),
+          :enable_starttls_auto => options.fetch("starttls", false),
+          :debug                => options.fetch("debug", false)
+        }
+      end
+    elsif @via == 'sendmail'
+      Mail.defaults do
+        delivery_method :sendmail
+      end
+    else
+      Mail.defaults do
+        delivery_method :@via, options
+      end
+    end # @via tests
+    @logger.debug("Email Output Registered!", :config => @config)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+      @logger.debug("Event being tested for Email", :tags => @tags, :event => event)
+      # Set Intersection - returns a new array with the items that are the same between the two
+      if !@tags.empty? && (event["tags"] & @tags).size == 0
+         # Skip events that have no tags in common with what we were configured
+         @logger.debug("No Tags match for Email Output!")
+         return
+      end
+
+    @logger.debug? && @logger.debug("Match data for Email - ", :match => @match)
+    successful = false
+    matchName = ""
+    operator = ""
+
+    # TODO(sissel): Delete this once match support is removed.
+    @match && @match.each do |name, query|
+      if successful
+        break
+      else
+        matchName = name
+      end
+      # now loop over the csv query
+      queryArray = query.split(',')
+      index = 1
+      while index < queryArray.length
+        field = queryArray.at(index -1)
+        value = queryArray.at(index)
+        index = index + 2
+        if field == ""
+          if value.downcase == "and"
+            operator = "and"
+          elsif value.downcase == "or"
+            operator = "or"
+          else
+            operator = "or"
+            @logger.error("Operator Provided Is Not Found, Currently We Only Support AND/OR Values! - defaulting to OR")
+          end
+        else
+          hasField = event[field]
+          @logger.debug? and @logger.debug("Does Event Contain Field - ", :hasField => hasField)
+          isValid = false
+          # if we have maching field and value is wildcard - we have a success
+          if hasField
+            if value == "*"
+              isValid = true
+            else
+              # we get an array so we need to loop over the values and find if we have a match
+              eventFieldValues = event[field]
+              @logger.debug? and @logger.debug("Event Field Values - ", :eventFieldValues => eventFieldValues)
+              eventFieldValues = [eventFieldValues] if not eventFieldValues.respond_to?(:each)
+              eventFieldValues.each do |eventFieldValue|
+                isValid = validateValue(eventFieldValue, value)
+                if isValid # no need to iterate any further
+                  @logger.debug("VALID CONDITION FOUND - ", :eventFieldValue => eventFieldValue, :value => value) 
+                  break
+                end
+              end # end eventFieldValues.each do
+            end # end value == "*"
+          end # end hasField
+          # if we have an AND operator and we have a successful == false break
+          if operator == "and" && !isValid
+            successful = false
+          elsif operator == "or" && (isValid || successful)
+            successful = true
+          else
+            successful = isValid
+          end
+        end
+      end
+    end # @match.each do
+
+    # The 'match' setting is deprecated and optional. If not set,
+    # default to success.
+    successful = true if @match.nil?
+
+    @logger.debug? && @logger.debug("Email Did we match any alerts for event : ", :successful => successful)
+
+    if successful
+      # first add our custom field - matchName - so we can use it in the sprintf function
+      event["matchName"] = matchName unless matchName.empty?
+      @logger.debug? and @logger.debug("Creating mail with these settings : ", :via => @via, :options => @options, :from => @from, :to => @to, :cc => @cc, :subject => @subject, :body => @body, :content_type => @contenttype, :htmlbody => @htmlbody, :attachments => @attachments, :to => to, :to => to)
+      formatedSubject = event.sprintf(@subject)
+      formattedBody = event.sprintf(@body)
+      formattedHtmlBody = event.sprintf(@htmlbody)
+      # we have a match(s) - send email
+      mail = Mail.new
+      mail.from = event.sprintf(@from)
+      mail.to = event.sprintf(@to)
+      if @replyto
+        mail.reply_to = event.sprintf(@replyto)
+      end
+      mail.cc = event.sprintf(@cc)
+      mail.subject = formatedSubject
+      if @htmlbody.empty?
+        formattedBody.gsub!(/\\n/, "\n") # Take new line in the email
+        mail.body = formattedBody
+      else
+        mail.text_part = Mail::Part.new do
+          content_type "text/plain; charset=UTF-8"
+          formattedBody.gsub!(/\\n/, "\n") # Take new line in the email
+          body formattedBody
+        end
+        mail.html_part = Mail::Part.new do
+          content_type "text/html; charset=UTF-8"
+          body formattedHtmlBody
+        end
+      end
+      @attachments.each do |fileLocation|
+        mail.add_file(fileLocation)
+      end # end @attachments.each
+      @logger.debug? and @logger.debug("Sending mail with these values : ", :from => mail.from, :to => mail.to, :cc => mail.cc, :subject => mail.subject)
+      mail.deliver!
+    end # end if successful
+  end # def receive
+
+
+  private
+  def validateValue(eventFieldValue, value)
+    valid = false
+    # order of this if-else is important - please don't change it
+    if value.start_with?(">=")# greater than or equal
+      value.gsub!(">=","")
+      if eventFieldValue.to_i >= value.to_i
+        valid = true
+      end
+    elsif value.start_with?("<=")# less than or equal
+      value.gsub!("<=","")
+      if eventFieldValue.to_i <= value.to_i
+        valid = true
+      end
+    elsif value.start_with?(">")# greater than
+      value.gsub!(">","")
+      if eventFieldValue.to_i > value.to_i
+        valid = true
+      end
+    elsif value.start_with?("<")# less than
+      value.gsub!("<","")
+      if eventFieldValue.to_i < value.to_i
+        valid = true
+      end
+    elsif value.start_with?("*")# contains
+      value.gsub!("*","")
+      if eventFieldValue.include?(value)
+        valid = true
+      end
+    elsif value.start_with?("!*")# does not contain
+      value.gsub!("!*","")
+      if !eventFieldValue.include?(value)
+        valid = true
+      end
+    elsif value.start_with?("!")# not equal
+      value.gsub!("!","")
+      if eventFieldValue != value
+        valid = true
+      end
+    else # default equal
+      if eventFieldValue == value
+        valid = true
+      end
+    end
+    return valid
+  end # end validateValue()
+
+end # class LogStash::Outputs::Email

data/lib/logstash/outputs/exec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+
+# This output will run a command for any matching event.
+#
+# Example:
+# 
+#     output {
+#       exec {
+#         type => abuse
+#         command => "iptables -A INPUT -s %{clientip} -j DROP"
+#       }
+#     }
+#
+# Run subprocesses via system ruby function
+#
+# WARNING: if you want it non-blocking you should use & or dtach or other such
+# techniques
+class LogStash::Outputs::Exec < LogStash::Outputs::Base
+
+  config_name "exec"
+  milestone 1
+
+  # Command line to execute via subprocess. Use dtach or screen to make it non blocking
+  config :command, :validate => :string, :required => true
+
+  public
+  def register
+    @logger.debug("exec output registered", :config => @config)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    @logger.debug("running exec command", :command => event.sprintf(@command))
+    system(event.sprintf(@command))
+  end # def receive
+
+end

data/lib/logstash/outputs/file.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/base"
+require "zlib"
+
+# File output.
+#
+# Write events to files on disk. You can use fields from the
+# event as parts of the filename.
+class LogStash::Outputs::File < LogStash::Outputs::Base
+
+  config_name "file"
+  milestone 2
+
+  # The path to the file to write. Event fields can be used here, 
+  # like "/var/log/logstash/%{host}/%{application}"
+  # One may also utilize the path option for date-based log 
+  # rotation via the joda time format. This will use the event
+  # timestamp.
+  # E.g.: path => "./test-%{+YYYY-MM-dd}.txt" to create 
+  # ./test-2013-05-29.txt 
+  config :path, :validate => :string, :required => true
+
+  # The maximum size of file to write. When the file exceeds this
+  # threshold, it will be rotated to the current filename + ".1"
+  # If that file already exists, the previous .1 will shift to .2
+  # and so forth.
+  #
+  # NOT YET SUPPORTED
+  config :max_size, :validate => :string
+
+  # The format to use when writing events to the file. This value
+  # supports any string and can include %{name} and other dynamic
+  # strings.
+  #
+  # If this setting is omitted, the full json representation of the
+  # event will be written as a single line.
+  config :message_format, :validate => :string
+
+  # Flush interval for flushing writes to log files. 0 will flush on every meesage
+  config :flush_interval, :validate => :number, :default => 2
+
+  # Gzip output stream
+  config :gzip, :validate => :boolean, :default => false
+
+  public
+  def register
+    require "fileutils" # For mkdir_p
+
+    workers_not_supported
+
+    @files = {}
+    now = Time.now
+    @last_flush_cycle = now
+    @last_stale_cleanup_cycle = now
+    flush_interval = @flush_interval.to_i
+    @stale_cleanup_interval = 10
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    path = event.sprintf(@path)
+    fd = open(path)
+
+    # TODO(sissel): Check if we should rotate the file.
+
+    if @message_format
+      output = event.sprintf(@message_format)
+    else
+      output = event.to_json
+    end
+
+    fd.write(output)
+    fd.write("\n")
+
+    flush(fd)
+    close_stale_files
+  end # def receive
+
+  def teardown
+    @logger.debug("Teardown: closing files")
+    @files.each do |path, fd|
+      begin
+        fd.close
+        @logger.debug("Closed file #{path}", :fd => fd)
+      rescue Exception => e
+        @logger.error("Excpetion while flushing and closing files.", :exception => e)
+      end
+    end
+    finished
+  end
+
+  private
+  def flush(fd)
+    if flush_interval > 0
+      flush_pending_files
+    else
+      fd.flush
+    end
+  end
+
+  # every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
+  def flush_pending_files
+    return unless Time.now - @last_flush_cycle >= flush_interval
+    @logger.debug("Starting flush cycle")
+    @files.each do |path, fd|
+      @logger.debug("Flushing file", :path => path, :fd => fd)
+      fd.flush
+    end
+    @last_flush_cycle = Time.now
+  end
+
+  # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
+  def close_stale_files
+    now = Time.now
+    return unless now - @last_stale_cleanup_cycle >= @stale_cleanup_interval
+    @logger.info("Starting stale files cleanup cycle", :files => @files)
+    inactive_files = @files.select { |path, fd| not fd.active }
+    @logger.debug("%d stale files found" % inactive_files.count, :inactive_files => inactive_files)
+    inactive_files.each do |path, fd|
+      @logger.info("Closing file %s" % path)
+      fd.close
+      @files.delete(path)
+    end
+    # mark all files as inactive, a call to write will mark them as active again
+    @files.each { |path, fd| fd.active = false }
+    @last_stale_cleanup_cycle = now
+  end
+
+  def open(path)
+    return @files[path] if @files.include?(path) and not @files[path].nil?
+
+    @logger.info("Opening file", :path => path)
+
+    dir = File.dirname(path)
+    if !Dir.exists?(dir)
+      @logger.info("Creating directory", :directory => dir)
+      FileUtils.mkdir_p(dir) 
+    end
+
+    # work around a bug opening fifos (bug JRUBY-6280)
+    stat = File.stat(path) rescue nil
+    if stat and stat.ftype == "fifo" and RUBY_PLATFORM == "java"
+      fd = java.io.FileWriter.new(java.io.File.new(path))
+    else
+      fd = File.new(path, "a")
+    end
+    if gzip
+      fd = Zlib::GzipWriter.new(fd)
+    end
+    @files[path] = IOWriter.new(fd)
+  end
+end # class LogStash::Outputs::File
+
+# wrapper class
+class IOWriter
+  def initialize(io)
+    @io = io
+  end
+  def write(*args)
+    @io.write(*args)
+    @active = true
+  end
+  def flush
+    @io.flush
+    if @io.class == Zlib::GzipWriter
+      @io.to_io.flush
+    end
+  end
+  def method_missing(method_name, *args, &block)
+    if @io.respond_to?(method_name)
+      @io.send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+  attr_accessor :active
+end