logstash-lib 1.3.2

data/lib/logstash/outputs/graphtastic.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# A plugin for a newly developed Java/Spring Metrics application
+# I didn't really want to code this project but I couldn't find
+# a respectable alternative that would also run on any Windows
+# machine - which is the problem and why I am not going with Graphite
+# and statsd.  This application provides multiple integration options
+# so as to make its use under your network requirements possible. 
+# This includes a REST option that is always enabled for your use
+# in case you want to write a small script to send the occasional 
+# metric data. 
+#
+# Find GraphTastic here : https://github.com/NickPadilla/GraphTastic
+class LogStash::Outputs::GraphTastic < LogStash::Outputs::Base
+  
+  config_name "graphtastic"
+  milestone 2
+  
+  # options are udp(fastest - default) - rmi(faster) - rest(fast) - tcp(don't use TCP yet - some problems - errors out on linux)
+  config :integration, :validate => ["udp","tcp","rmi","rest"], :default => "udp"
+  
+  # if using rest as your end point you need to also provide the application url
+  # it defaults to localhost/graphtastic.  You can customize the application url
+  # by changing the name of the .war file.  There are other ways to change the 
+  # application context, but they vary depending on the Application Server in use.
+  # Please consult your application server documentation for more on application
+  # contexts.
+  config :context, :validate => :string, :default => "graphtastic"
+  
+  # metrics hash - you will provide a name for your metric and the metric 
+  # data as key value pairs.  so for example:
+  #
+  # metrics => { "Response" => "%{response}" } 
+  #
+  # example for the logstash config
+  #
+  # metrics => [ "Response", "%{response}" ]
+  #
+  # NOTE: you can also use the dynamic fields for the key value as well as the actual value
+  config :metrics, :validate => :hash, :default => {}
+   
+  # host for the graphtastic server - defaults to 127.0.0.1
+  config :host, :validate => :string, :default => "127.0.0.1"
+  
+  # port for the graphtastic instance - defaults to 1199 for RMI, 1299 for TCP, 1399 for UDP, and 8080 for REST
+  config :port, :validate => :number
+  
+  # number of attempted retry after send error - currently only way to integrate
+  # errored transactions - should try and save to a file or later consumption
+  # either by graphtastic utility or by this program after connectivity is
+  # ensured to be established. 
+  config :retries, :validate => :number, :default => 1
+  
+  # the number of metrics to send to GraphTastic at one time. 60 seems to be the perfect 
+  # amount for UDP, with default packet size. 
+  config :batch_number, :validate => :number, :default => 60
+  
+  # setting allows you to specify where we save errored transactions
+  # this makes the most sense at this point - will need to decide
+  # on how we reintegrate these error metrics
+  # NOT IMPLEMENTED!
+  config :error_file, :validate => :string, :default => ""
+  
+  public
+   def register
+     @batch = []
+     begin
+       if @integration.downcase == "rmi"
+         if RUBY_ENGINE != "jruby"
+            raise Exception.new("LogStash::Outputs::GraphTastic# JRuby is needed for RMI to work!")
+         end
+         require "java"
+         if @port.nil?
+           @port = 1199
+         end
+         registry = java.rmi.registry.LocateRegistry.getRegistry(@host, @port);
+         @remote = registry.lookup("RmiMetricService")
+       elsif @integration.downcase == "rest"
+         require "net/http"         
+         if @port.nil?
+           @port = 8080
+           gem "mail" #outputs/email, # License: MIT License
+         end
+         @http = Net::HTTP.new(@host, @port)
+       end
+       @logger.info("GraphTastic Output Successfully Registered! Using #{@integration} Integration!")
+     rescue 
+       @logger.error("*******ERROR :  #{$!}")
+     end
+   end
+
+  public
+  def receive(event)
+    return unless output?(event)
+    # Set Intersection - returns a new array with the items that are the same between the two
+    if !@tags.empty? && (event["tags"] & @tags).size == 0
+       # Skip events that have no tags in common with what we were configured
+       @logger.debug("No Tags match for GraphTastic Output!")
+       return
+    end
+    @retry = 1
+    @logger.debug("Event found for GraphTastic!", :tags => @tags, :event => event)
+    @metrics.each do |name, metric|
+      postMetric(event.sprintf(name),event.sprintf(metric),(event["@timestamp"]*1000))# unix_timestamp is what I need in seconds - multiply by 1000 to make milliseconds.
+    end
+  end
+  
+  def postMetric(name, metric, timestamp)
+    message = name+","+metric+","+timestamp.to_s
+    if @batch.length < @batch_number
+      @batch.push(message)
+    else
+      flushMetrics()      
+    end    
+  end
+  
+  def flushMetrics()
+    begin
+      if @integration.downcase == "tcp"
+        flushViaTCP()
+      elsif @integration.downcase == "rmi"
+        flushViaRMI() 
+      elsif @integration.downcase == "udp"
+        flushViaUDP()
+      elsif @integration.downcase == "rest"
+        flushViaREST()
+      else
+        @logger.error("GraphTastic Not Able To Find Correct Integration - Nothing Sent - Integration Type : ", :@integration => @integration)
+      end
+      @batch.clear
+    rescue
+      @logger.error("*******ERROR :  #{$!}")
+      @logger.info("*******Attempting #{@retry} out of #{@retries}")
+      while @retry < @retries
+        @retry = @retry + 1
+        flushMetrics()
+      end
+    end
+  end
+  
+  # send metrics via udp
+  def flushViaUDP()
+    if @port.nil?
+     @port = 1399
+    end
+    udpsocket.send(@batch.join(','), 0, @host, @port)
+    @logger.debug("GraphTastic Sent Message Using UDP : #{@batch.join(',')}")
+  end
+  
+  # send metrics via REST
+  def flushViaREST()
+    request = Net::HTTP::Put.new("/#{@context}/addMetric/#{@batch.join(',')}")
+    response = @http.request(request)
+    if response == 'ERROR'
+      raise 'Error happend when sending metric to GraphTastic using REST!'
+    end
+    @logger.debug("GraphTastic Sent Message Using REST : #{@batch.join(',')}", :response => response.inspect)    
+  end
+  
+  # send metrics via RMI
+  def flushViaRMI()
+    if RUBY_ENGINE != "jruby"
+       raise Exception.new("LogStash::Outputs::GraphTastic# JRuby is needed for RMI to work!")
+    end
+    @remote.insertMetrics(@batch.join(','))
+    @logger.debug("GraphTastic Sent Message Using RMI : #{@batch.join(',')}")
+  end
+  
+  # send metrics via tcp
+  def flushViaTCP()
+    # to correctly read the line we need to ensure we send \r\n at the end of every message.
+    if @port.nil?
+      @port = 1299
+    end
+    tcpsocket = TCPSocket.open(@host, @port)
+    tcpsocket.send(@batch.join(',')+"\r\n", 0)
+    tcpsocket.close
+    @logger.debug("GraphTastic Sent Message Using TCP : #{@batch.join(',')}")
+  end
+
+  def udpsocket; @socket ||= UDPSocket.new end
+  
+end

data/lib/logstash/outputs/hipchat.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/outputs/http"
+
+# This output allows you to write events to [HipChat](https://www.hipchat.com/).
+#
+class LogStash::Outputs::HipChat < LogStash::Outputs::Base
+
+  config_name "hipchat"
+  milestone 1
+
+  # The HipChat authentication token.
+  config :token, :validate => :string, :required => true
+
+  # The ID or name of the room.
+  config :room_id, :validate => :string, :required => true
+
+  # The name the message will appear be sent from.
+  config :from, :validate => :string, :default => "logstash"
+
+  # Whether or not this message should trigger a notification for people in the room.
+  config :trigger_notify, :validate => :boolean, :default => false
+
+  # Background color for message.
+  # HipChat currently supports one of "yellow", "red", "green", "purple",
+  # "gray", or "random". (default: yellow)
+  config :color, :validate => :string, :default => "yellow"
+
+  # Message format to send, event tokens are usable here.
+  config :format, :validate => :string, :default => "%{message}"
+
+  public
+  def register
+    require "ftw"
+    require "uri"
+
+    @agent = FTW::Agent.new
+
+    @url = "https://api.hipchat.com/v1/rooms/message?auth_token=" + @token
+    @content_type = "application/x-www-form-urlencoded"
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    hipchat_data = Hash.new
+    hipchat_data['room_id'] = @room_id
+    hipchat_data['from']    = @from
+    hipchat_data['color']   = @color
+    hipchat_data['notify']  = @trigger_notify ? "1" : "0"
+    hipchat_data['message'] = event.sprintf(@format)
+
+    @logger.debug("HipChat data", :hipchat_data => hipchat_data)
+
+    begin
+      request = @agent.post(@url)
+      request["Content-Type"] = @content_type
+      request.body = encode(hipchat_data)
+
+      response = @agent.execute(request)
+
+      # Consume body to let this connection be reused
+      rbody = ""
+      response.read_body { |c| rbody << c }
+      #puts rbody
+    rescue Exception => e
+      @logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
+    end
+  end # def receive
+
+  # shamelessly lifted this from the LogStash::Outputs::Http, I'd rather put this
+  # in a common place for both to use, but unsure where that place is or should be
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value)
+    end.join("&")
+  end # def encode
+
+end # class LogStash::Outputs::HipChat

data/lib/logstash/outputs/http.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+class LogStash::Outputs::Http < LogStash::Outputs::Base
+  # This output lets you `PUT` or `POST` events to a
+  # generic HTTP(S) endpoint
+  #
+  # Additionally, you are given the option to customize
+  # the headers sent as well as basic customization of the
+  # event json itself.
+
+  config_name "http"
+  milestone 1
+
+  # URL to use
+  config :url, :validate => :string, :required => :true
+
+  # validate SSL?
+  config :verify_ssl, :validate => :boolean, :default => true
+
+  # What verb to use
+  # only put and post are supported for now
+  config :http_method, :validate => ["put", "post"], :required => :true
+
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]
+  config :headers, :validate => :hash
+
+  # Content type
+  #
+  # If not specified, this defaults to the following:
+  #
+  # * if format is "json", "application/json"
+  # * if format is "form", "application/x-www-form-urlencoded"
+  config :content_type, :validate => :string
+
+  # This lets you choose the structure and parts of the event that are sent.
+  #
+  #
+  # For example:
+  #
+  #    mapping => ["foo", "%{host}", "bar", "%{type}"]
+  config :mapping, :validate => :hash
+
+  # Set the format of the http body.
+  #
+  # If form, then the body will be the mapping (or whole event) converted
+  # into a query parameter string (foo=bar&baz=fizz...)
+  #
+  # If message, then the body will be the result of formatting the event according to message
+  #
+  # Otherwise, the event is sent as json.
+  config :format, :validate => ["json", "form", "message"], :default => "json"
+
+  config :message, :validate => :string
+
+  public
+  def register
+    require "ftw"
+    require "uri"
+    @agent = FTW::Agent.new
+    # TODO(sissel): SSL verify mode?
+
+    if @content_type.nil?
+      case @format
+        when "form" ; @content_type = "application/x-www-form-urlencoded"
+        when "json" ; @content_type = "application/json"
+      end
+    end
+    if @format == "message"
+      if @message.nil?
+        raise "message must be set if message format is used"
+      end
+      if @content_type.nil?
+        raise "content_type must be set if message format is used"
+      end
+      unless @mapping.nil?
+        @logger.warn "mapping is not supported and will be ignored if message format is used"
+      end
+    end
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    if @mapping
+      evt = Hash.new
+      @mapping.each do |k,v|
+        evt[k] = event.sprintf(v)
+      end
+    else
+      evt = event.to_hash
+    end
+
+    case @http_method
+    when "put"
+      request = @agent.put(event.sprintf(@url))
+    when "post"
+      request = @agent.post(event.sprintf(@url))
+    else
+      @logger.error("Unknown verb:", :verb => @http_method)
+    end
+    
+    if @headers
+      @headers.each do |k,v|
+        request.headers[k] = event.sprintf(v)
+      end
+    end
+
+    request["Content-Type"] = @content_type
+
+    begin
+      if @format == "json"
+        request.body = evt.to_json
+      elsif @format == "message"
+        request.body = event.sprintf(@message)
+      else
+        request.body = encode(evt)
+      end
+      #puts "#{request.port} / #{request.protocol}"
+      #puts request
+      #puts 
+      #puts request.body
+      response = @agent.execute(request)
+
+      # Consume body to let this connection be reused
+      rbody = ""
+      response.read_body { |c| rbody << c }
+      #puts rbody
+    rescue Exception => e
+      @logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
+    end
+  end # def receive
+
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value)
+    end.join("&")
+  end # def encode
+end

data/lib/logstash/outputs/irc.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "thread"
+
+# Write events to IRC
+#
+class LogStash::Outputs::Irc < LogStash::Outputs::Base
+
+  config_name "irc"
+  milestone 1
+
+  # Address of the host to connect to
+  config :host, :validate => :string, :required => true
+
+  # Port on host to connect to.
+  config :port, :validate => :number, :default => 6667
+
+  # IRC Nickname
+  config :nick, :validate => :string, :default => "logstash"
+
+  # IRC Username
+  config :user, :validate => :string, :default => "logstash"
+
+  # IRC Real name
+  config :real, :validate => :string, :default => "logstash"
+
+  # IRC server password
+  config :password, :validate => :password
+
+  # Channels to broadcast to.
+  # 
+  # These should be full channel names including the '#' symbol, such as
+  # "#logstash".
+  config :channels, :validate => :array, :required => true
+
+  # Message format to send, event tokens are usable here
+  config :format, :validate => :string, :default => "%{message}"
+
+  # Set this to true to enable SSL.
+  config :secure, :validate => :boolean, :default => false
+
+  # Limit the rate of messages sent to IRC in messages per second.
+  config :messages_per_second, :validate => :number, :default => 0.5
+
+  public
+  def register
+    require "cinch"
+    @irc_queue = Queue.new
+    @logger.info("Connecting to irc server", :host => @host, :port => @port, :nick => @nick, :channels => @channels)
+
+    @bot = Cinch::Bot.new
+    @bot.loggers.clear
+    @bot.configure do |c|
+      c.server = @host
+      c.port = @port
+      c.nick = @nick
+      c.user = @user
+      c.realname = @real
+      c.channels = @channels
+      c.password = @password.value rescue nil
+      c.ssl.use = @secure
+      c.messages_per_second = @messages_per_second if @messages_per_second
+    end
+    Thread.new(@bot) do |bot|
+      bot.start
+    end
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    @logger.debug("Sending message to channels", :event => event)
+    text = event.sprintf(@format)
+    @bot.channels.each do |channel|
+      @logger.debug("Sending to...", :channel => channel, :text => text)
+      channel.msg(text)
+    end # channels.each
+  end # def receive
+end # class LogStash::Outputs::Irc