logstash-lib 1.3.2

data/lib/logstash/outputs/sns.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
+
+# SNS output.
+#
+# Send events to Amazon's Simple Notification Service, a hosted pub/sub
+# framework.  It supports subscribers of type email, HTTP/S, SMS, and SQS.
+#
+# For further documentation about the service see:
+#
+#   http://docs.amazonwebservices.com/sns/latest/api/
+#
+# This plugin looks for the following fields on events it receives:
+#
+#  * `sns` - If no ARN is found in the configuration file, this will be used as
+#  the ARN to publish.
+#  * `sns_subject` - The subject line that should be used.
+#  Optional. The "%{host}" will be used if not present and truncated at
+#  `MAX_SUBJECT_SIZE_IN_CHARACTERS`.
+#  * `sns_message` - The message that should be
+#  sent. Optional. The event serialzed as JSON will be used if not present and
+#  with the @message truncated so that the length of the JSON fits in
+#  `MAX_MESSAGE_SIZE_IN_BYTES`.
+#
+class LogStash::Outputs::Sns < LogStash::Outputs::Base
+  include LogStash::PluginMixins::AwsConfig
+
+  MAX_SUBJECT_SIZE_IN_CHARACTERS  = 100
+  MAX_MESSAGE_SIZE_IN_BYTES       = 32768
+
+  config_name "sns"
+  milestone 1
+
+  # Message format.  Defaults to plain text.
+  config :format, :validate => [ "json", "plain" ], :default => "plain"
+
+  # SNS topic ARN.
+  config :arn, :validate => :string
+
+  # When an ARN for an SNS topic is specified here, the message
+  # "Logstash successfully booted" will be sent to it when this plugin
+  # is registered.
+  #
+  # Example: arn:aws:sns:us-east-1:770975001275:logstash-testing
+  #
+  config :publish_boot_message_arn, :validate => :string
+
+  public
+  def aws_service_endpoint(region)
+    return {
+        :sns_endpoint => "sns.#{region}.amazonaws.com"
+    }
+  end
+
+  public
+  def register
+    require "aws-sdk"
+
+    @sns = AWS::SNS.new(aws_options_hash)
+
+    # Try to publish a "Logstash booted" message to the ARN provided to
+    # cause an error ASAP if the credentials are bad.
+    if @publish_boot_message_arn
+      @sns.topics[@publish_boot_message_arn].publish("Logstash successfully booted", :subject => "Logstash booted")
+    end
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    arn     = Array(event["sns"]).first || @arn
+
+    raise "An SNS ARN required." unless arn
+
+    message = Array(event["sns_message"]).first
+    subject = Array(event["sns_subject"]).first || event.source
+
+    # Ensure message doesn't exceed the maximum size.
+    if message
+      # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
+      message = message.slice(0, MAX_MESSAGE_SIZE_IN_BYTES)
+    else
+      if @format == "plain"
+        message = self.class.format_message(event)
+      else
+        message = self.class.json_message(event)
+      end
+    end
+
+    # Log event.
+    @logger.debug("Sending event to SNS topic [#{arn}] with subject [#{subject}] and message:")
+    message.split("\n").each { |line| @logger.debug(line) }
+
+    # Publish the message.
+    @sns.topics[arn].publish(message, :subject => subject.slice(0, MAX_SUBJECT_SIZE_IN_CHARACTERS))
+  end
+
+  def self.json_message(event)
+    json      = event.to_json
+    json_size = json.bytesize
+
+    # Truncate only the message if the JSON structure is too large.
+    if json_size > MAX_MESSAGE_SIZE_IN_BYTES
+      # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
+      event["message"] = event["message"].slice(0, (event["message"].bytesize - (json_size - MAX_MESSAGE_SIZE_IN_BYTES)))
+    end
+
+    event.to_json
+  end
+
+  def self.format_message(event)
+    message =  "Date: #{event["@timestamp"]}\n"
+    message << "Source: #{event["source"]}\n"
+    message << "Tags: #{event["tags"].join(', ')}\n"
+    message << "Fields: #{event.to_hash.inspect}\n"
+    message << "Message: #{event["message"]}"
+
+    # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
+    message.slice(0, MAX_MESSAGE_SIZE_IN_BYTES)
+  end
+end

data/lib/logstash/outputs/solr_http.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "stud/buffer"
+require "rubygems"
+require "uuidtools"
+
+# This output lets you index&store your logs in Solr. If you want to get
+# started quickly you should use version 4.4 or above in schemaless mode,
+# which will try and guess your fields automatically. To turn that on,
+# you can use the example included in the Solr archive:
+#
+#     tar zxf solr-4.4.0.tgz
+#     cd example
+#     mv solr solr_ #back up the existing sample conf
+#     cp -r example-schemaless/solr/ .  #put the schemaless conf in place
+#     java -jar start.jar   #start Solr
+#
+# You can learn more about Solr at <https://lucene.apache.org/solr/>
+
+class LogStash::Outputs::SolrHTTP < LogStash::Outputs::Base
+  include Stud::Buffer
+
+  config_name "solr_http"
+
+  milestone 1
+
+  # URL used to connect to Solr
+  config :solr_url, :validate => :string, :default => "http://localhost:8983/solr"
+
+  # Number of events to queue up before writing to Solr
+  config :flush_size, :validate => :number, :default => 100
+
+  # Amount of time since the last flush before a flush is done even if
+  # the number of buffered events is smaller than flush_size
+  config :idle_flush_time, :validate => :number, :default => 1
+
+  # Solr document ID for events. You'd typically have a variable here, like
+  # '%{foo}' so you can assign your own IDs
+  config :document_id, :validate => :string, :default => nil
+
+  public
+  def register
+    require "rsolr"
+    @solr = RSolr.connect :url => @solr_url
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+  end #def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    buffer_receive(event)
+  end #def receive
+
+  public
+  def flush(events, teardown=false)
+    documents = []  #this is the array of hashes that we push to Solr as documents
+
+    events.each do |event|
+        document = event.to_hash()
+        document["@timestamp"] = document["@timestamp"].iso8601 #make the timestamp ISO
+        if @document_id.nil?
+          document ["id"] = UUIDTools::UUID.random_create    #add a unique ID
+        else
+          document ["id"] = event.sprintf(@document_id)      #or use the one provided
+        end
+        documents.push(document)
+    end
+
+    @solr.add(documents)
+    rescue Exception => e
+      @logger.warn("An error occurred while indexing: #{e.message}")
+  end #def flush
+end #class LogStash::Outputs::SolrHTTP

data/lib/logstash/outputs/sqs.rb

@@ -0,0 +1,141 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
+require "stud/buffer"
+require "digest/sha2"
+
+# Push events to an Amazon Web Services Simple Queue Service (SQS) queue.
+#
+# SQS is a simple, scalable queue system that is part of the 
+# Amazon Web Services suite of tools.
+#
+# Although SQS is similar to other queuing systems like AMQP, it
+# uses a custom API and requires that you have an AWS account.
+# See http://aws.amazon.com/sqs/ for more details on how SQS works,
+# what the pricing schedule looks like and how to setup a queue.
+#
+# To use this plugin, you *must*:
+#
+#  * Have an AWS account
+#  * Setup an SQS queue
+#  * Create an identify that has access to publish messages to the queue.
+#
+# The "consumer" identity must have the following permissions on the queue:
+#
+#  * sqs:ChangeMessageVisibility
+#  * sqs:ChangeMessageVisibilityBatch
+#  * sqs:GetQueueAttributes
+#  * sqs:GetQueueUrl
+#  * sqs:ListQueues
+#  * sqs:SendMessage
+#  * sqs:SendMessageBatch
+#
+# Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
+# A sample policy is as follows:
+#
+#      {
+#        "Statement": [
+#          {
+#            "Sid": "Stmt1347986764948",
+#            "Action": [
+#              "sqs:ChangeMessageVisibility",
+#              "sqs:ChangeMessageVisibilityBatch",
+#              "sqs:DeleteMessage",
+#              "sqs:DeleteMessageBatch",
+#              "sqs:GetQueueAttributes",
+#              "sqs:GetQueueUrl",
+#              "sqs:ListQueues",
+#              "sqs:ReceiveMessage"
+#            ],
+#            "Effect": "Allow",
+#            "Resource": [
+#              "arn:aws:sqs:us-east-1:200850199751:Logstash"
+#            ]
+#          }
+#        ]
+#      }
+#
+# See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
+#
+class LogStash::Outputs::SQS < LogStash::Outputs::Base
+  include LogStash::PluginMixins::AwsConfig
+  include Stud::Buffer
+
+  config_name "sqs"
+  milestone 1
+
+  # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN.
+  config :queue, :validate => :string, :required => true
+
+  # Set to true if you want send messages to SQS in batches with batch_send
+  # from the amazon sdk
+  config :batch, :validate => :boolean, :default => true
+
+  # If batch is set to true, the number of events we queue up for a batch_send.
+  config :batch_events, :validate => :number, :default => 10
+
+  # If batch is set to true, the maximum amount of time between batch_send commands when there are pending events to flush.
+  config :batch_timeout, :validate => :number, :default => 5
+
+  public
+  def aws_service_endpoint(region)
+    return {
+        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
+    }
+  end
+
+  public 
+  def register
+    require "aws-sdk"
+
+    @sqs = AWS::SQS.new(aws_options_hash)
+
+    if @batch
+      if @batch_events > 10
+        raise RuntimeError.new(
+          "AWS only allows a batch_events parameter of 10 or less"
+        )
+      elsif @batch_events <= 1
+        raise RuntimeError.new(
+          "batch_events parameter must be greater than 1 (or its not a batch)"
+        )
+      end
+      buffer_initialize(
+        :max_items => @batch_events,
+        :max_interval => @batch_timeout,
+        :logger => @logger
+      )
+    end
+
+    begin
+      @logger.debug("Connecting to AWS SQS queue '#{@queue}'...")
+      @sqs_queue = @sqs.queues.named(@queue)
+    rescue Exception => e
+      @logger.error("Unable to access SQS queue '#{@queue}': #{e.to_s}")
+    end # begin/rescue
+
+    @logger.info("Connected to AWS SQS queue '#{@queue}' successfully.")
+  end # def register
+
+  public
+  def receive(event)
+    if @batch
+      buffer_receive(event.to_json)
+      return
+    end
+    @sqs_queue.send_message(event.to_json)
+  end # def receive
+
+  # called from Stud::Buffer#buffer_flush when there are events to flush
+  def flush(events, teardown=false)
+    @sqs_queue.batch_send(events)
+  end
+
+  public
+  def teardown
+    buffer_flush(:final => true)
+    @sqs_queue = nil
+    finished
+  end # def teardown
+end

data/lib/logstash/outputs/statsd.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# statsd is a server for aggregating counters and other metrics to ship to
+# graphite.
+#
+# The most basic coverage of this plugin is that the 'namespace', 'sender', and
+# 'metric' names are combined into the full metric path like so:
+#
+#     namespace.sender.metric
+#
+# The general idea is that you send statsd count or latency data and every few
+# seconds it will emit the aggregated values to graphite (aggregates like
+# average, max, stddev, etc)
+#
+# You can learn about statsd here:
+#
+# * <http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/>
+# * <https://github.com/etsy/statsd>
+#
+# A simple example usage of this is to count HTTP hits by response code; to learn
+# more about that, check out the 
+# [log metrics tutorial](../tutorials/metrics-from-logs)
+class LogStash::Outputs::Statsd < LogStash::Outputs::Base
+  ## Regex stolen from statsd code
+  RESERVED_CHARACTERS_REGEX = /[\:\|\@]/
+  config_name "statsd"
+  milestone 2
+
+  # The address of the Statsd server.
+  config :host, :validate => :string, :default => "localhost"
+
+  # The port to connect to on your statsd server.
+  config :port, :validate => :number, :default => 8125
+
+  # The statsd namespace to use for this metric
+  config :namespace, :validate => :string, :default => "logstash"
+
+  # The name of the sender.
+  # Dots will be replaced with underscores
+  config :sender, :validate => :string, :default => "%{host}"
+
+  # An increment metric. metric names as array.
+  config :increment, :validate => :array, :default => []
+
+  # A decrement metric. metric names as array.
+  config :decrement, :validate => :array, :default => []
+
+  # A timing metric. metric_name => duration as hash
+  config :timing, :validate => :hash, :default => {}
+
+  # A count metric. metric_name => count as hash
+  config :count, :validate => :hash, :default => {}
+
+  # A set metric. metric_name => string to append as hash
+  config :set, :validate => :hash, :default => {}
+
+  # A gauge metric. metric_name => gauge as hash
+  config :gauge, :validate => :hash, :default => {}
+  
+  # The sample rate for the metric
+  config :sample_rate, :validate => :number, :default => 1
+
+  # The final metric sent to statsd will look like the following (assuming defaults)
+  # logstash.sender.file_name
+  #
+  # Enable debugging output?
+  config :debug, :validate => :boolean, :default => false
+
+  public
+  def register
+    require "statsd"
+    @client = Statsd.new(@host, @port)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    @client.namespace = event.sprintf(@namespace) if not @namespace.empty?
+    @logger.debug? and @logger.debug("Original sender: #{@sender}")
+    sender = event.sprintf(@sender)
+    @logger.debug? and @logger.debug("Munged sender: #{sender}")
+    @logger.debug? and @logger.debug("Event: #{event}")
+    @increment.each do |metric|
+      @client.increment(build_stat(event.sprintf(metric), sender), @sample_rate)
+    end
+    @decrement.each do |metric|
+      @client.decrement(build_stat(event.sprintf(metric), sender), @sample_rate)
+    end
+    @count.each do |metric, val|
+      @client.count(build_stat(event.sprintf(metric), sender),
+                    event.sprintf(val).to_f, @sample_rate)
+    end
+    @timing.each do |metric, val|
+      @client.timing(build_stat(event.sprintf(metric), sender),
+                     event.sprintf(val).to_f, @sample_rate)
+    end
+    @set.each do |metric, val|
+      @client.set(build_stat(event.sprintf(metric), sender),
+                    event.sprintf(val), @sample_rate)
+    end
+    @gauge.each do |metric, val|
+      @client.gauge(build_stat(event.sprintf(metric), sender),
+                    event.sprintf(val).to_f, @sample_rate)
+    end
+  end # def receive
+
+  def build_stat(metric, sender=@sender)
+    sender = sender.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_').gsub(".", "_")
+    metric = metric.gsub('::','.').gsub(RESERVED_CHARACTERS_REGEX, '_')
+    @logger.debug? and @logger.debug("Formatted value", :sender => sender, :metric => metric)
+    return "#{sender}.#{metric}"
+  end
+end # class LogStash::Outputs::Statsd