logstash-lib 1.3.2

data/lib/logstash/inputs/snmptrap.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# Read snmp trap messages as events
+#
+# Resulting @message looks like :
+#   #<SNMP::SNMPv1_Trap:0x6f1a7a4 @varbind_list=[#<SNMP::VarBind:0x2d7bcd8f @value="teststring", 
+#   @name=[1.11.12.13.14.15]>], @timestamp=#<SNMP::TimeTicks:0x1af47e9d @value=55>, @generic_trap=6, 
+#   @enterprise=[1.2.3.4.5.6], @source_ip="127.0.0.1", @agent_addr=#<SNMP::IpAddress:0x29a4833e @value="\xC0\xC1\xC2\xC3">, 
+#   @specific_trap=99>
+#
+
+class LogStash::Inputs::Snmptrap < LogStash::Inputs::Base
+  config_name "snmptrap"
+  milestone 1
+
+  # The address to listen on
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on. Remember that ports less than 1024 (privileged
+  # ports) may require root to use. hence the default of 1062.
+  config :port, :validate => :number, :default => 1062
+
+  # SNMP Community String to listen for.
+  config :community, :validate => :string, :default => "public"
+
+  # directory of YAML MIB maps  (same format ruby-snmp uses)
+  config :yamlmibdir, :validate => :string
+
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    require "snmp"
+    @snmptrap = nil
+    if @yamlmibdir
+      @logger.info("checking #{@yamlmibdir} for MIBs")
+      Dir["#{@yamlmibdir}/*.yaml"].each do |yamlfile|
+        mib_name = File.basename(yamlfile, ".*")
+        @yaml_mibs ||= []
+        @yaml_mibs << mib_name
+      end
+      @logger.info("found MIBs: #{@yaml_mibs.join(',')}") if @yaml_mibs
+    end
+  end # def register
+
+  public
+  def run(output_queue)
+    begin
+      # snmp trap server
+      snmptrap_listener(output_queue)
+    rescue => e
+      @logger.warn("SNMP Trap listener died", :exception => e, :backtrace => e.backtrace)
+      sleep(5)
+      retry
+    end # begin
+  end # def run
+
+  private
+  def snmptrap_listener(output_queue)
+    traplistener_opts = {:Port => @port, :Community => @community, :Host => @host}
+    if !@yaml_mibs.empty?
+      traplistener_opts.merge!({:MibDir => @yamlmibdir, :MibModules => @yaml_mibs})
+    end
+    @logger.info("It's a Trap!", traplistener_opts.dup)
+    @snmptrap = SNMP::TrapListener.new(traplistener_opts)
+
+    @snmptrap.on_trap_default do |trap|
+      begin
+        event = LogStash::Event.new("message" => trap.inspect, "host" => trap.source_ip)
+        decorate(event)
+        trap.each_varbind do |vb|
+          event[vb.name.to_s] = vb.value.to_s
+        end
+        @logger.debug("SNMP Trap received: ", :trap_object => trap.inspect)
+        output_queue << event
+      rescue => event
+        @logger.error("Failed to create event", :trap_object => trap.inspect)
+      end
+    end
+    @snmptrap.join
+  end # def snmptrap_listener
+
+end # class LogStash::Inputs::Snmptrap

data/lib/logstash/inputs/sqlite.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Read rows from an sqlite database.
+#
+# This is most useful in cases where you are logging directly to a table.
+# Any tables being watched must have an 'id' column that is monotonically
+# increasing.
+#
+# All tables are read by default except:
+# * ones matching 'sqlite_%' - these are internal/adminstrative tables for sqlite
+# * 'since_table' - this is used by this plugin to track state.
+#
+# ## Example
+# 
+#     % sqlite /tmp/example.db
+#     sqlite> CREATE TABLE weblogs (
+#         id INTEGER PRIMARY KEY AUTOINCREMENT,
+#         ip STRING,
+#         request STRING,
+#         response INTEGER);
+#     sqlite> INSERT INTO weblogs (ip, request, response) 
+#         VALUES ("1.2.3.4", "/index.html", 200);
+#
+# Then with this logstash config:
+#
+#     input {
+#       sqlite {
+#         path => "/tmp/example.db"
+#         type => weblogs
+#       }
+#     }
+#     output {
+#       stdout {
+#         debug => true
+#       }
+#     }
+#
+# Sample output:
+#
+#     {
+#       "@source"      => "sqlite://sadness/tmp/x.db",
+#       "@tags"        => [],
+#       "@fields"      => {
+#         "ip"       => "1.2.3.4",
+#         "request"  => "/index.html",
+#         "response" => 200
+#       },
+#       "@timestamp"   => "2013-05-29T06:16:30.850Z",
+#       "@source_host" => "sadness",
+#       "@source_path" => "/tmp/x.db",
+#       "@message"     => "",
+#       "@type"        => "foo"
+#     }
+#
+class LogStash::Inputs::Sqlite < LogStash::Inputs::Base
+  config_name "sqlite"
+  milestone 1
+
+  # The path to the sqlite database file.
+  config :path, :validate => :string, :required => true
+
+  # Any tables to exclude by name.
+  # By default all tables are followed.
+  config :exclude_tables, :validate => :array, :default => []
+
+  # How many rows to fetch at a time from each SELECT call.
+  config :batch, :validate => :number, :default => 5
+
+  SINCE_TABLE = :since_table
+
+  public
+  def init_placeholder_table(db)
+    begin
+      db.create_table SINCE_TABLE do 
+        String :table
+        Int    :place
+      end
+    rescue
+      @logger.debug("since tables already exists")
+    end
+  end
+
+  public
+  def get_placeholder(db, table)
+    since = db[SINCE_TABLE]
+    x = since.where(:table => "#{table}")
+    if x[:place].nil?
+      init_placeholder(db, table) 
+      return 0
+    else
+      @logger.debug("placeholder already exists, it is #{x[:place]}")
+      return x[:place][:place]
+    end
+  end
+
+  public 
+  def init_placeholder(db, table)
+    @logger.debug("init placeholder for #{table}")
+    since = db[SINCE_TABLE]
+    since.insert(:table => table, :place => 0)
+  end
+
+  public
+  def update_placeholder(db, table, place)
+    @logger.debug("set placeholder to #{place}")
+    since = db[SINCE_TABLE]
+    since.where(:table => table).update(:place => place)
+  end
+
+  public 
+  def get_all_tables(db)
+    return db["SELECT * FROM sqlite_master WHERE type = 'table' AND tbl_name != '#{SINCE_TABLE}' AND tbl_name NOT LIKE 'sqlite_%'"].map { |t| t[:name] }.select { |n| !@exclude_tables.include?(n) }
+  end
+  
+  public
+  def get_n_rows_from_table(db, table, offset, limit)
+    dataset = db["SELECT * FROM #{table}"]
+    return db["SELECT * FROM #{table} WHERE (id > #{offset}) ORDER BY 'id' LIMIT #{limit}"].map { |row| row }
+  end
+  
+  public
+  def register
+    require "sequel"
+    require "jdbc/sqlite3" 
+    @host = Socket.gethostname
+    @logger.info("Registering sqlite input", :database => @path)
+    @db = Sequel.connect("jdbc:sqlite:#{@path}") 
+    @tables = get_all_tables(@db)
+    @table_data = {}
+    @tables.each do |table|
+      init_placeholder_table(@db)
+      last_place = get_placeholder(@db, table)
+      @table_data[table] = { :name => table, :place => last_place }
+    end
+  end # def register
+
+  public
+  def run(queue)
+    sleep_min = 0.01
+    sleep_max = 5
+    sleeptime = sleep_min
+
+    begin
+      @logger.debug("Tailing sqlite db", :path => @path)
+      loop do
+        count = 0
+        @table_data.each do |k, table|
+          table_name = table[:name]
+          offset = table[:place]
+          @logger.debug("offset is #{offset}", :k => k, :table => table_name)
+          rows = get_n_rows_from_table(@db, table_name, offset, @batch)
+          count += rows.count
+          rows.each do |row| 
+            event = LogStash::Event.new("host" => @host, "db" => @db)
+            decorate(event)
+            # store each column as a field in the event.
+            row.each do |column, element|
+              next if column == :id
+              event[column.to_s] = element
+            end
+            queue << event
+            @table_data[k][:place] = row[:id]
+          end
+          # Store the last-seen row in the database
+          update_placeholder(@db, table_name, @table_data[k][:place])
+        end
+
+        if count == 0
+          # nothing found in that iteration
+          # sleep a bit
+          @logger.debug("No new rows. Sleeping.", :time => sleeptime)
+          sleeptime = [sleeptime * 2, sleep_max].min
+          sleep(sleeptime)
+        else
+          sleeptime = sleep_min
+        end
+      end # loop
+    end # begin/rescue
+  end #run
+
+end # class Logtstash::Inputs::EventLog
+

data/lib/logstash/inputs/sqs.rb

@@ -0,0 +1,172 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
+require "digest/sha2"
+
+# Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
+#
+# SQS is a simple, scalable queue system that is part of the 
+# Amazon Web Services suite of tools.
+#
+# Although SQS is similar to other queuing systems like AMQP, it
+# uses a custom API and requires that you have an AWS account.
+# See http://aws.amazon.com/sqs/ for more details on how SQS works,
+# what the pricing schedule looks like and how to setup a queue.
+#
+# To use this plugin, you *must*:
+#
+#  * Have an AWS account
+#  * Setup an SQS queue
+#  * Create an identify that has access to consume messages from the queue.
+#
+# The "consumer" identity must have the following permissions on the queue:
+#
+#  * sqs:ChangeMessageVisibility
+#  * sqs:ChangeMessageVisibilityBatch
+#  * sqs:DeleteMessage
+#  * sqs:DeleteMessageBatch
+#  * sqs:GetQueueAttributes
+#  * sqs:GetQueueUrl
+#  * sqs:ListQueues
+#  * sqs:ReceiveMessage
+#
+# Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
+# A sample policy is as follows:
+#
+#     {
+#       "Statement": [
+#         {
+#           "Action": [
+#             "sqs:ChangeMessageVisibility",
+#             "sqs:ChangeMessageVisibilityBatch",
+#             "sqs:GetQueueAttributes",
+#             "sqs:GetQueueUrl",
+#             "sqs:ListQueues",
+#             "sqs:SendMessage",
+#             "sqs:SendMessageBatch"
+#           ],
+#           "Effect": "Allow",
+#           "Resource": [
+#             "arn:aws:sqs:us-east-1:123456789012:Logstash"
+#           ]
+#         }
+#       ]
+#     } 
+#
+# See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
+#
+class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
+  include LogStash::PluginMixins::AwsConfig
+
+  config_name "sqs"
+  milestone 1
+
+  default :codec, "json"
+
+  # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
+  config :queue, :validate => :string, :required => true
+
+  # Name of the event field in which to store the SQS message ID
+  config :id_field, :validate => :string
+
+  # Name of the event field in which to store the SQS message MD5 checksum
+  config :md5_field, :validate => :string
+
+  # Name of the event field in which to store the  SQS message Sent Timestamp
+  config :sent_timestamp_field, :validate => :string
+
+  public
+  def aws_service_endpoint(region)
+    return {
+        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
+    }
+  end
+
+  public
+  def register
+    @logger.info("Registering SQS input", :queue => @queue)
+    require "aws-sdk"
+
+    @sqs = AWS::SQS.new(aws_options_hash)
+
+    begin
+      @logger.debug("Connecting to AWS SQS queue", :queue => @queue)
+      @sqs_queue = @sqs.queues.named(@queue)
+      @logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
+    rescue Exception => e
+      @logger.error("Unable to access SQS queue.", :error => e.to_s, :queue => @queue)
+      throw e
+    end # begin/rescue
+  end # def register
+
+  public
+  def run(output_queue)
+    @logger.debug("Polling SQS queue", :queue => @queue)
+
+    receive_opts = {
+        :limit => 10,
+        :visibility_timeout => 30,
+        :attributes => [:sent_at]
+    }
+
+    continue_polling = true
+    while running? && continue_polling
+      continue_polling = run_with_backoff(60, 1) do
+        @sqs_queue.receive_message(receive_opts) do |message|
+          if message
+            @codec.decode(message.body) do |event|
+              decorate(event)
+              if @id_field
+                event[@id_field] = message.id
+              end
+              if @md5_field
+                event[@md5_field] = message.md5
+              end
+              if @sent_timestamp_field
+                event[@sent_timestamp_field] = message.sent_timestamp.utc
+              end
+              @logger.debug? && @logger.debug("Processed SQS message", :message_id => message.id, :message_md5 => message.md5, :sent_timestamp => message.sent_timestamp, :queue => @queue)
+              output_queue << event
+              message.delete
+            end # codec.decode
+          end # valid SQS message
+        end # receive_message
+      end # run_with_backoff
+    end # polling loop
+  end # def run
+
+  def teardown
+    @sqs_queue = nil
+    finished
+  end # def teardown
+
+  private
+  # Runs an AWS request inside a Ruby block with an exponential backoff in case
+  # we exceed the allowed AWS RequestLimit.
+  #
+  # @param [Integer] max_time maximum amount of time to sleep before giving up.
+  # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
+  # @param [Block] block Ruby code block to execute.
+  def run_with_backoff(max_time, sleep_time, &block)
+    if sleep_time > max_time
+      @logger.error("AWS::EC2::Errors::RequestLimitExceeded ... failed.", :queue => @queue)
+      return false
+    end # retry limit exceeded
+
+    begin
+      block.call
+    rescue AWS::EC2::Errors::RequestLimitExceeded
+      @logger.info("AWS::EC2::Errors::RequestLimitExceeded ... retrying SQS request", :queue => @queue, :sleep_time => sleep_time)
+      sleep sleep_time
+      run_with_backoff(max_time, sleep_time * 2, &block)
+    rescue AWS::EC2::Errors::InstanceLimitExceeded
+      @logger.warn("AWS::EC2::Errors::InstanceLimitExceeded ... aborting SQS message retreival.")
+      return false
+    rescue Exception => bang
+      @logger.error("Error reading SQS queue.", :error => bang, :queue => @queue)
+      return false
+    end # begin/rescue
+    return true
+  end # def run_with_backoff
+end # class LogStash::Inputs::SQS

data/lib/logstash/inputs/stdin.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+# Read events from standard input.
+#
+# By default, each event is assumed to be one line. If you
+# want to join lines, you'll want to use the multiline filter.
+class LogStash::Inputs::Stdin < LogStash::Inputs::Base
+  config_name "stdin"
+  milestone 3
+
+  default :codec, "line"
+
+  public
+  def register
+    @host = Socket.gethostname
+  end # def register
+
+  def run(queue) 
+    while true
+      begin
+        # Based on some testing, there is no way to interrupt an IO.sysread nor
+        # IO.select call in JRuby. Bummer :(
+        data = $stdin.sysread(16384)
+        @codec.decode(data) do |event|
+          decorate(event)
+          event["host"] = @host
+          queue << event
+        end
+      rescue EOFError, LogStash::ShutdownSignal
+        # stdin closed or a requested shutdown
+        break
+      end
+    end # while true
+    finished
+  end # def run
+
+  public
+  def teardown
+    @logger.debug("stdin shutting down.")
+    $stdin.close rescue nil
+    finished
+  end # def teardown
+end # class LogStash::Inputs::Stdin