logstash-lib 1.3.2

data/lib/logstash/outputs/jira.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+# Origin https://groups.google.com/forum/#!msg/logstash-users/exgrB4iQ-mw/R34apku5nXsJ
+# and https://botbot.me/freenode/logstash/msg/4169496/ 
+# via https://gist.github.com/electrical/4660061e8fff11cdcf37#file-jira-rb
+
+# Uses jiralicious as the bridge to JIRA
+# By Martin Cleaver, Blended Perspectives
+# with a lot of help from 'electrical' in #logstash
+
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+#
+# This is so is most useful so you can use logstash to parse and structure
+# your logs and ship structured, json events to JIRA
+#
+# To use this, you'll need to ensure your JIRA instance allows REST calls
+
+class LogStash::Outputs::Jira < LogStash::Outputs::Base
+  config_name "jira"
+  milestone 2
+
+  # The hostname to send logs to. This should target your JIRA server 
+  # and has to have the REST interface enabled
+  config :host, :validate => :string
+
+  config :username, :validate => :string, :required => true
+  config :password, :validate => :string, :required => true
+
+  # Javalicious has no proxy support
+###
+  # JIRA Project number
+  config :projectid, :validate => :string, :required => true
+
+  # JIRA Issuetype number
+  config :issuetypeid, :validate => :string, :required => true
+
+  # JIRA Summary
+  config :summary, :validate => :string, :required => true
+
+  # JIRA Priority
+  config :priority, :validate => :string, :required => true
+
+  # JIRA Reporter
+  config :reporter, :validate => :string
+
+  # JIRA Reporter
+  config :assignee, :validate => :string
+
+### The following have not been implemented
+  # Ticket creation method
+  #config :method, :validate => :string, :default => 'new'
+  
+  # Search fields; When in 'append' method. search for a ticket that has these fields and data.
+  #config :searchfields, :validate => :hash
+  
+  # createfields; Add data to these fields at initial creation
+  #config :createfields, :validate => :hash
+  
+  # appendfields; Update data in these fields when appending data to an existing ticket
+  #config :appendfields, :validate => :hash
+  
+  # Comment; Add this in the comment field ( is for new and append method the same )
+  #config :comment, :validate => :string
+
+
+  public
+  def register
+    require "jiralicious" # 0.2.2 works for me
+    # nothing to do
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    if event == LogStash::SHUTDOWN
+      finished
+      return
+    end
+
+    Jiralicious.configure do |config|
+      config.username = @username
+      config.password = @password
+      config.uri = @host
+      config.auth_type = :cookie
+      config.api_version = "latest"
+    end
+
+
+issue = Jiralicious::Issue.new
+    issue.fields.set_id("project", @projectid) # would have prefered a project key, https://github.com/jstewart/jiralicious/issues/16
+    issue.fields.set("summary", @summary)
+    issue.fields.set_id("issuetype", @issuetypeid)
+    issue.fields.set_name("reporter", @reporter)
+    issue.fields.set_name("assignee", @assignee)
+    issue.fields.set_id("priority", @priority)
+#puts issue.fields.to_yaml
+    issue.save
+
+
+
+#    if response.is_a?(Net::HTTPSuccess)
+#      @logger.info("Event send to JIRA OK!")
+#    else
+#      @logger.warn("HTTP error", :error => response.error!)
+#    end
+  end # def receive
+end # class LogStash::Outputs::Jira

data/lib/logstash/outputs/juggernaut.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/event"
+
+# Push messages to the juggernaut websockets server:
+#
+# * https://github.com/maccman/juggernaut
+#
+# Wraps Websockets and supports other methods (including xhr longpolling) This
+# is basically, just an extension of the redis output (Juggernaut pulls
+# messages from redis).  But it pushes messages to a particular channel and
+# formats the messages in the way juggernaut expects.
+class LogStash::Outputs::Juggernaut < LogStash::Outputs::Base
+
+  config_name "juggernaut"
+  milestone 1
+
+  # The hostname of the redis server to which juggernaut is listening.
+  config :host, :validate => :string, :default => "127.0.0.1"
+
+  # The port to connect on.
+  config :port, :validate => :number, :default => 6379
+
+  # The redis database number.
+  config :db, :validate => :number, :default => 0
+
+  # Redis initial connection timeout in seconds.
+  config :timeout, :validate => :number, :default => 5
+
+  # Password to authenticate with.  There is no authentication by default.
+  config :password, :validate => :password
+
+  # List of channels to which to publish. Dynamic names are
+  # valid here, for example "logstash-%{type}".
+  config :channels, :validate => :array, :required => true
+
+  # How should the message be formatted before pushing to the websocket.
+  config :message_format, :validate => :string
+
+  public
+  def register
+    require 'redis'
+
+    if not @channels
+      raise RuntimeError.new(
+        "Must define the channels on which to publish the messages"
+      )
+    end
+    # end TODO
+
+    @redis = nil
+  end # def register
+
+  private
+  def connect
+    Redis.new(
+      :host => @host,
+      :port => @port,
+      :timeout => @timeout,
+      :db => @db,
+      :password => @password
+    )
+  end # def connect
+
+  # A string used to identify a redis instance in log messages
+  private
+  def identity
+    @name || "redis://#{@password}@#{@host}:#{@port}/#{@db} #{@data_type}:#{@key}"
+  end
+
+
+  public
+  def receive(event)
+    return unless output?(event)
+    begin
+      @redis ||= connect
+      if @message_format
+        formatted = event.sprintf(@message_format)
+      else
+        formatted = event.to_json
+      end
+      juggernaut_message = {
+        "channels" => @channels.collect{ |x| event.sprintf(x) },
+        "data" => event["message"]
+      }
+
+      @redis.publish 'juggernaut', juggernaut_message.to_json
+    rescue => e
+      @logger.warn("Failed to send event to redis", :event => event,
+                   :identity => identity, :exception => e,
+                   :backtrace => e.backtrace)
+      raise e
+    end
+  end # def receive
+
+  public
+  def teardown
+    if @data_type == 'channel' and @redis
+      @redis.quit
+      @redis = nil
+    end
+  end
+
+end

data/lib/logstash/outputs/librato.rb

@@ -0,0 +1,146 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+class LogStash::Outputs::Librato < LogStash::Outputs::Base
+  # This output lets you send metrics, annotations and alerts to
+  # Librato based on Logstash events
+  # 
+  # This is VERY experimental and inefficient right now.
+
+  config_name "librato"
+  milestone 1
+
+  # Your Librato account
+  # usually an email address
+  config :account_id, :validate => :string, :required => true
+
+  # Your Librato API Token
+  config :api_token, :validate => :string, :required => true
+
+  # Gauges
+  # Send data to Librato as a gauge
+  #
+  # Example:
+  #   ["value", "%{bytes_recieved}", "source", "%{host}", "name", "apache_bytes"]
+  # Additionally, you can override the `measure_time` for the event. Must be a unix timestamp:
+  #   ["value", "%{bytes_recieved}", "source", "%{host}", "name", "apache_bytes","measure_time", "%{my_unixtime_field}]
+  # Default is to use the event's timestamp
+  config :gauge, :validate => :hash, :default => {}
+
+  # Counters
+  # Send data to Librato as a counter
+  #
+  # Example:
+  #   ["value", "1", "source", "%{host}", "name", "messages_received"]
+  # Additionally, you can override the `measure_time` for the event. Must be a unix timestamp:
+  #   ["value", "1", "source", "%{host}", "name", "messages_received", "measure_time", "%{my_unixtime_field}"]
+  # Default is to use the event's timestamp
+  config :counter, :validate => :hash, :default => {}
+
+  # Annotations
+  # Registers an annotation with Librato
+  # The only required field is `title` and `name`.
+  # `start_time` and `end_time` will be set to `event["@timestamp"].to_i`
+  # You can add any other optional annotation values as well.
+  # All values will be passed through `event.sprintf`
+  #
+  # Example:
+  #   ["title":"Logstash event on %{host}", "name":"logstash_stream"]
+  # or
+  #   ["title":"Logstash event", "description":"%{message}", "name":"logstash_stream"]
+  config :annotation, :validate => :hash, :default => {}
+
+  # Batch size
+  # Number of events to batch up before sending to Librato.
+  #
+  config :batch_size, :validate => :string, :default => "10"
+
+  public
+  def register
+    require "net/https"
+    require "uri"
+    @url = "https://metrics-api.librato.com/v1/"
+    @uri = URI.parse(@url)
+    @client = Net::HTTP.new(@uri.host, @uri.port)
+    @client.use_ssl = true
+    @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    
+  end # def register
+
+  public
+  def receive(event)
+    # TODO (lusis)
+    # batch and flush
+    return unless output?(event)
+
+    metrics_event = Hash.new
+    unless @gauge.size == 0
+      g_hash = Hash[*@gauge.collect{|k,v| [k,event.sprintf(v)]}.flatten]
+      g_hash.each do |k,v|
+        g_hash[k] = v.to_f if k=="value"
+      end
+      g_hash['measure_time'] = event["@timestamp"].to_i unless g_hash['measure_time']
+      @logger.warn("Gauges hash", :data => g_hash)
+      metrics_event['gauges'] = Array.new 
+      metrics_event['gauges'] << g_hash
+      @logger.warn("Metrics hash", :data => metrics_event)
+    end
+    unless @counter.size == 0
+      c_hash = Hash[*@counter.collect{|k,v| [k,event.sprintf(v)]}.flatten]
+      c_hash.each do |k,v|
+        c_hash[k] = v.to_f if k=="value"
+      end
+      c_hash['measure_time'] = event["@timestamp"].to_i unless c_hash['measure_time']
+      @logger.warn("Counters hash", :data => c_hash)
+      metrics_event['counters'] = Array.new
+      metrics_event['counters'] << c_hash
+      @logger.warn("Metrics hash", :data => metrics_event)
+    end
+   
+    # TODO (lusis)
+    # Clean this mess up
+    unless metrics_event.size == 0
+      request = Net::HTTP::Post.new(@uri.path+"metrics")
+      request.basic_auth(@account_id, @api_token)
+      
+      begin
+        request.body = metrics_event.to_json
+        request.add_field("Content-Type", 'application/json')
+        response = @client.request(request)
+        @logger.warn("Librato convo", :request => request.inspect, :response => response.inspect)
+        raise unless response.code == '200'
+      rescue Exception => e
+        @logger.warn("Unhandled exception", :request => request.inspect, :response => response.inspect, :exception => e.inspect)
+      end
+    end
+
+    unless @annotation.size == 0
+      annotation_hash = Hash.new
+      annotation_hash['annotations'] = Array.new
+      @logger.warn("Original Annotation", :data => @annotation)
+      annotation_event = Hash[*@annotation.collect{|k,v| [event.sprintf(k),event.sprintf(v)]}.flatten]
+      @logger.warn("Annotation event", :data => annotation_event)
+      
+      annotation_path = "#{@uri.path}annotations/#{annotation_event['name']}"
+      @logger.warn("Annotation path", :data => annotation_path)
+      request = Net::HTTP::Post.new(annotation_path)
+      request.basic_auth(@account_id, @api_token)
+      annotation_event.delete('name')
+      annotation_event['start_time'] = event["@timestamp"].to_i unless annotation_event['start_time']
+      annotation_event['end_time'] = event["@timestamp"].to_i unless annotation_event['end_time']
+      annotation_hash['annotations'] << annotation_event
+      @logger.warn("Annotation event", :data => annotation_event)
+
+      begin
+        request.body = annotation_event.to_json
+        request.add_field("Content-Type", 'application/json')
+        response = @client.request(request)
+        @logger.warn("Librato convo", :request => request.inspect, :response => response.inspect)
+        raise unless response.code == '201'
+      rescue Exception => e
+        @logger.warn("Unhandled exception", :request => request.inspect, :response => response.inspect, :exception => e.inspect)
+      end
+    end
+  end # def receive
+end

data/lib/logstash/outputs/loggly.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "uri"
+# TODO(sissel): Move to something that performs better than net/http
+require "net/http"
+require "net/https"
+
+# Ugly monkey patch to get around <http://jira.codehaus.org/browse/JRUBY-5529>
+Net::BufferedIO.class_eval do
+    BUFSIZE = 1024 * 16
+
+    def rbuf_fill
+      timeout(@read_timeout) {
+        @rbuf << @io.sysread(BUFSIZE)
+      }
+    end
+end
+
+# Got a loggly account? Use logstash to ship logs to Loggly!
+#
+# This is most useful so you can use logstash to parse and structure
+# your logs and ship structured, json events to your account at Loggly.
+#
+# To use this, you'll need to use a Loggly input with type 'http'
+# and 'json logging' enabled.
+class LogStash::Outputs::Loggly < LogStash::Outputs::Base
+  config_name "loggly"
+  milestone 2
+
+  # The hostname to send logs to. This should target the loggly http input
+  # server which is usually "logs.loggly.com"
+  config :host, :validate => :string, :default => "logs.loggly.com"
+
+  # The loggly http input key to send to.
+  # This is usually visible in the Loggly 'Inputs' page as something like this
+  #     https://logs.hoover.loggly.net/inputs/abcdef12-3456-7890-abcd-ef0123456789
+  #                                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  #                                           \---------->   key   <-------------/
+  #
+  # You can use %{foo} field lookups here if you need to pull the api key from
+  # the event. This is mainly aimed at multitenant hosting providers who want
+  # to offer shipping a customer's logs to that customer's loggly account.
+  config :key, :validate => :string, :required => true
+
+  # Should the log action be sent over https instead of plain http
+  config :proto, :validate => :string, :default => "http"
+
+  # Proxy Host
+  config :proxy_host, :validate => :string
+
+  # Proxy Port
+  config :proxy_port, :validate => :number
+
+  # Proxy Username
+  config :proxy_user, :validate => :string
+
+  # Proxy Password
+  config :proxy_password, :validate => :password, :default => ""
+
+
+  public
+  def register
+    # nothing to do
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    if event == LogStash::SHUTDOWN
+      finished
+      return
+    end
+
+    # Send the event over http.
+    url = URI.parse("#{@proto}://#{@host}/inputs/#{event.sprintf(@key)}")
+    @logger.info("Loggly URL", :url => url)
+    http = Net::HTTP::Proxy(@proxy_host, @proxy_port, @proxy_user, @proxy_password.value).new(url.host, url.port)
+    if url.scheme == 'https'
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    request = Net::HTTP::Post.new(url.path)
+    request.body = event.to_json
+    response = http.request(request)
+    if response.is_a?(Net::HTTPSuccess)
+      @logger.info("Event send to Loggly OK!")
+    else
+      @logger.warn("HTTP error", :error => response.error!)
+    end
+  end # def receive
+end # class LogStash::Outputs::Loggly