logstash-lib 1.3.2

data/lib/logstash/inputs/redis.rb

@@ -0,0 +1,263 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+
+# Read events from a redis. Supports both redis channels and also redis lists
+# (using BLPOP)
+#
+# For more information about redis, see <http://redis.io/>
+#
+# ## `batch_count` note
+#
+# If you use the 'batch_count' setting, you *must* use a redis version 2.6.0 or
+# newer. Anything older does not support the operations used by batching.
+class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
+  config_name "redis"
+  milestone 2
+
+  default :codec, "json"
+
+  # Name is used for logging in case there are multiple instances.
+  # This feature has no real function and will be removed in future versions.
+  config :name, :validate => :string, :default => "default", :deprecated => true
+
+  # The hostname of your redis server.
+  config :host, :validate => :string, :default => "127.0.0.1"
+
+  # The port to connect on.
+  config :port, :validate => :number, :default => 6379
+
+  # The redis database number.
+  config :db, :validate => :number, :default => 0
+
+  # Initial connection timeout in seconds.
+  config :timeout, :validate => :number, :default => 5
+
+  # Password to authenticate with. There is no authentication by default.
+  config :password, :validate => :password
+
+  # The name of the redis queue (we'll use BLPOP against this).
+  # TODO: remove soon.
+  config :queue, :validate => :string, :deprecated => true
+
+  # The name of a redis list or channel.
+  # TODO: change required to true
+  config :key, :validate => :string, :required => false
+
+  # Either list or channel.  If redis\_type is list, then we will BLPOP the
+  # key.  If redis\_type is channel, then we will SUBSCRIBE to the key.
+  # If redis\_type is pattern_channel, then we will PSUBSCRIBE to the key.
+  # TODO: change required to true
+  config :data_type, :validate => [ "list", "channel", "pattern_channel" ], :required => false
+
+  # How many events to return from redis using EVAL
+  config :batch_count, :validate => :number, :default => 1
+
+  public
+  def register
+    require 'redis'
+    @redis = nil
+    @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
+
+    # TODO remove after setting key and data_type to true
+    if @queue
+      if @key or @data_type
+        raise RuntimeError.new(
+          "Cannot specify queue parameter and key or data_type"
+        )
+      end
+      @key = @queue
+      @data_type = 'list'
+    end
+
+    if not @key or not @data_type
+      raise RuntimeError.new(
+        "Must define queue, or key and data_type parameters"
+      )
+    end
+    # end TODO
+
+    @logger.info("Registering redis", :identity => identity)
+  end # def register
+
+  # A string used to identify a redis instance in log messages
+  # TODO(sissel): Use instance variables for this once the @name config
+  # option is removed.
+  private
+  def identity
+    @name || "#{@redis_url} #{@data_type}:#{@key}"
+  end
+
+  private
+  def connect
+    redis = Redis.new(
+      :host => @host,
+      :port => @port,
+      :timeout => @timeout,
+      :db => @db,
+      :password => @password.nil? ? nil : @password.value
+    )
+    load_batch_script(redis) if @data_type == 'list' && (@batch_count > 1)
+    return redis
+  end # def connect
+
+  private
+  def load_batch_script(redis)
+    #A redis lua EVAL script to fetch a count of keys
+    #in case count is bigger than current items in queue whole queue will be returned without extra nil values
+    redis_script = <<EOF
+          local i = tonumber(ARGV[1])
+          local res = {}
+          local length = redis.call('llen',KEYS[1])
+          if length < i then i = length end
+          while (i > 0) do
+            local item = redis.call("lpop", KEYS[1])
+            if (not item) then
+              break
+            end
+            table.insert(res, item)
+            i = i-1
+          end
+          return res
+EOF
+    @redis_script_sha = redis.script(:load, redis_script)
+  end
+
+  private
+  def queue_event(msg, output_queue)
+    begin
+      @codec.decode(msg) do |event|
+        decorate(event)
+        output_queue << event
+      end
+    rescue => e # parse or event creation error
+      @logger.error("Failed to create event", :message => msg, :exception => e,
+                    :backtrace => e.backtrace);
+    end
+  end
+
+  private
+  def list_listener(redis, output_queue)
+
+    # blpop returns the 'key' read from as well as the item result
+    # we only care about the result (2nd item in the list).
+    item = redis.blpop(@key, 0)[1]
+
+    # blpop failed or .. something?
+    # TODO(sissel): handle the error
+    return if item.nil?
+    queue_event(item, output_queue)
+
+    # If @batch_count is 1, there's no need to continue.
+    return if @batch_count == 1
+
+    begin
+      redis.evalsha(@redis_script_sha, [@key], [@batch_count-1]).each do |item|
+        queue_event(item, output_queue)
+      end
+
+      # Below is a commented-out implementation of 'batch fetch'
+      # using pipelined LPOP calls. This in practice has been observed to
+      # perform exactly the same in terms of event throughput as
+      # the evalsha method. Given that the EVALSHA implementation uses
+      # one call to redis instead of N (where N == @batch_count) calls,
+      # I decided to go with the 'evalsha' method of fetching N items
+      # from redis in bulk.
+      #redis.pipelined do
+        #error, item = redis.lpop(@key)
+        #(@batch_count-1).times { redis.lpop(@key) }
+      #end.each do |item|
+        #queue_event(item, output_queue) if item
+      #end
+      # --- End commented out implementation of 'batch fetch'
+    rescue Redis::CommandError => e
+      if e.to_s =~ /NOSCRIPT/ then
+        @logger.warn("Redis may have been restarted, reloading redis batch EVAL script", :exception => e);
+        load_batch_script(redis)
+        retry
+      else
+        raise e
+      end
+    end
+  end
+
+  private
+  def channel_listener(redis, output_queue)
+    redis.subscribe @key do |on|
+      on.subscribe do |channel, count|
+        @logger.info("Subscribed", :channel => channel, :count => count)
+      end
+
+      on.message do |channel, message|
+        queue_event message, output_queue
+      end
+
+      on.unsubscribe do |channel, count|
+        @logger.info("Unsubscribed", :channel => channel, :count => count)
+      end
+    end
+  end
+
+  private
+  def pattern_channel_listener(redis, output_queue)
+    redis.psubscribe @key do |on|
+      on.psubscribe do |channel, count|
+        @logger.info("Subscribed", :channel => channel, :count => count)
+      end
+
+      on.pmessage do |ch, event, message|
+        queue_event message, output_queue
+      end
+
+      on.punsubscribe do |channel, count|
+        @logger.info("Unsubscribed", :channel => channel, :count => count)
+      end
+    end
+  end
+
+  # Since both listeners have the same basic loop, we've abstracted the outer
+  # loop.
+  private
+  def listener_loop(listener, output_queue)
+    while !finished?
+      begin
+        @redis ||= connect
+        self.send listener, @redis, output_queue
+      rescue Redis::CannotConnectError => e
+        @logger.warn("Redis connection problem", :exception => e)
+        sleep 1
+        @redis = connect
+      rescue => e # redis error
+        @logger.warn("Failed to get event from redis", :name => @name,
+                     :exception => e, :backtrace => e.backtrace)
+        raise e
+      end
+    end # while !finished?
+  end # listener_loop
+
+  public
+  def run(output_queue)
+    if @data_type == 'list'
+      listener_loop :list_listener, output_queue
+    elsif @data_type == 'channel'
+      listener_loop :channel_listener, output_queue
+    else
+      listener_loop :pattern_channel_listener, output_queue
+    end
+  end # def run
+
+  public
+  def teardown
+    if @data_type == 'channel' and @redis
+      @redis.unsubscribe
+      @redis.quit
+      @redis = nil
+    end
+    if @data_type == 'pattern_channel' and @redis
+      @redis.punsubscribe
+      @redis.quit
+      @redis = nil
+    end
+  end
+end # class LogStash::Inputs::Redis

data/lib/logstash/inputs/relp.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/util/relp"
+require "logstash/util/socket_peer"
+
+
+# Read RELP events over a TCP socket.
+#
+# For more information about RELP, see 
+# <http://www.rsyslog.com/doc/imrelp.html>
+#
+# This protocol implements application-level acknowledgements to help protect
+# against message loss.
+#
+# Message acks only function as far as messages being put into the queue for
+# filters; anything lost after that point will not be retransmitted
+class LogStash::Inputs::Relp < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+
+  config_name "relp"
+  milestone 1
+
+  default :codec, "plain"
+
+  # The address to listen on.
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on.
+  config :port, :validate => :number, :required => true
+
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    @logger.info("Starting relp input listener", :address => "#{@host}:#{@port}")
+    @relp_server = RelpServer.new(@host, @port,['syslog'])
+  end # def register
+
+  private
+  def relp_stream(relpserver,socket,output_queue,client_address)
+    loop do
+      frame = relpserver.syslog_read(socket)
+      @codec.decode(frame["message"]) do |event|
+        decorate(event)
+        event["host"] = client_address
+        output_queue << event
+      end
+
+      #To get this far, the message must have made it into the queue for 
+      #filtering. I don't think it's possible to wait for output before ack
+      #without fundamentally breaking the plugin architecture
+      relpserver.ack(socket, frame['txnr'])
+    end
+  end
+
+  public
+  def run(output_queue)
+    @thread = Thread.current
+    loop do
+      begin
+        # Start a new thread for each connection.
+        Thread.start(@relp_server.accept) do |client|
+            rs = client[0]
+            socket = client[1]
+            # monkeypatch a 'peer' method onto the socket.
+            socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+            peer = socket.peer
+            @logger.debug("Relp Connection to #{peer} created")
+          begin
+            relp_stream(rs,socket, output_queue, peer)
+          rescue Relp::ConnectionClosed => e
+            @logger.debug("Relp Connection to #{peer} Closed")
+          rescue Relp::RelpError => e
+            @logger.warn('Relp error: '+e.class.to_s+' '+e.message)
+            #TODO: Still not happy with this, are they all warn level?
+            #Will this catch everything I want it to?
+            #Relp spec says to close connection on error, ensure this is the case
+          end
+        end # Thread.start
+      rescue Relp::InvalidCommand,Relp::InappropriateCommand => e
+        @logger.warn('Relp client trying to open connection with something other than open:'+e.message)
+      rescue Relp::InsufficientCommands
+        @logger.warn('Relp client incapable of syslog')
+      rescue IOError, Interrupted
+        if @interrupted
+          # Intended shutdown, get out of the loop
+          @relp_server.shutdown
+          break
+        else
+          # Else it was a genuine IOError caused by something else, so propagate it up..
+          raise
+        end
+      end
+    end # loop
+  end # def run
+
+  def teardown
+    @interrupted = true
+    @thread.raise(Interrupted.new)
+  end
+end # class LogStash::Inputs::Relp
+
+#TODO: structured error logging

data/lib/logstash/inputs/s3.rb

@@ -0,0 +1,279 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "time"
+require "tmpdir"
+
+# Stream events from files from a S3 bucket.
+#
+# Each line from each file generates an event.
+# Files ending in '.gz' are handled as gzip'ed files.
+class LogStash::Inputs::S3 < LogStash::Inputs::Base
+  config_name "s3"
+  milestone 1
+
+  # TODO(sissel): refactor to use 'line' codec (requires removing both gzip
+  # support and readline usage). Support gzip through a gzip codec! ;)
+  default :codec, "plain"
+
+  # The credentials of the AWS account used to access the bucket.
+  # Credentials can be specified:
+  # - As an ["id","secret"] array
+  # - As a path to a file containing AWS_ACCESS_KEY_ID=... and AWS_SECRET_ACCESS_KEY=...
+  # - In the environment (variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
+  config :credentials, :validate => :array, :default => nil
+
+  # The name of the S3 bucket.
+  config :bucket, :validate => :string, :required => true
+
+  # The AWS region for your bucket.
+  config :region, :validate => ["us-east-1", "us-west-1", "us-west-2",
+                                "eu-west-1", "ap-southeast-1", "ap-southeast-2",
+                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"],
+                                :deprecated => "'region' has been deprecated in favor of 'region_endpoint'"
+
+  # The AWS region for your bucket.
+  config :region_endpoint, :validate => ["us-east-1", "us-west-1", "us-west-2",
+                                "eu-west-1", "ap-southeast-1", "ap-southeast-2",
+                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => "us-east-1"
+
+  # If specified, the prefix the filenames in the bucket must match (not a regexp)
+  config :prefix, :validate => :string, :default => nil
+
+  # Where to write the since database (keeps track of the date
+  # the last handled file was added to S3). The default will write
+  # sincedb files to some path matching "$HOME/.sincedb*"
+  config :sincedb_path, :validate => :string, :default => nil
+
+  # Name of a S3 bucket to backup processed files to.
+  config :backup_to_bucket, :validate => :string, :default => nil
+
+  # Path of a local directory to backup processed files to.
+  config :backup_to_dir, :validate => :string, :default => nil
+
+  # Whether to delete processed files from the original bucket.
+  config :delete, :validate => :boolean, :default => false
+
+  # Interval to wait between to check the file list again after a run is finished.
+  # Value is in seconds.
+  config :interval, :validate => :number, :default => 60
+
+  public
+  def register
+    require "digest/md5"
+    require "aws-sdk"
+
+    @region_endpoint = @region if !@region.empty?
+
+    @logger.info("Registering s3 input", :bucket => @bucket, :region_endpoint => @region_endpoint)
+
+    if @credentials.nil?
+      @access_key_id = ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+    elsif @credentials.is_a? Array
+      if @credentials.length ==1
+        File.open(@credentials[0]) { |f| f.each do |line|
+          unless (/^\#/.match(line))
+            if(/\s*=\s*/.match(line))
+              param, value = line.split('=', 2)
+              param = param.chomp().strip()
+              value = value.chomp().strip()
+              if param.eql?('AWS_ACCESS_KEY_ID')
+                @access_key_id = value
+              elsif param.eql?('AWS_SECRET_ACCESS_KEY')
+                @secret_access_key = value
+              end
+            end
+          end
+        end
+        }
+      elsif @credentials.length == 2
+        @access_key_id = @credentials[0]
+        @secret_access_key = @credentials[1]
+      else
+        raise ArgumentError.new('Credentials must be of the form "/path/to/file" or ["id", "secret"]')
+      end
+    end
+    if @access_key_id.nil? or @secret_access_key.nil?
+      raise ArgumentError.new('Missing AWS credentials')
+    end
+
+    if @bucket.nil?
+      raise ArgumentError.new('Missing AWS bucket')
+    end
+
+    if @sincedb_path.nil?
+      if ENV['HOME'].nil?
+        raise ArgumentError.new('No HOME or sincedb_path set')
+      end
+      @sincedb_path = File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
+    end
+
+    s3 = AWS::S3.new(
+      :access_key_id => @access_key_id,
+      :secret_access_key => @secret_access_key,
+      :region => @region_endpoint
+    )
+
+    @s3bucket = s3.buckets[@bucket]
+
+    unless @backup_to_bucket.nil?
+      @backup_bucket = s3.buckets[@backup_to_bucket]
+      unless @backup_bucket.exists?
+        s3.buckets.create(@backup_to_bucket)
+      end
+    end
+
+    unless @backup_to_dir.nil?
+      Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
+    end
+
+  end # def register
+
+  public
+  def run(queue)
+    loop do
+      process_new(queue)
+      sleep(@interval)
+    end
+    finished
+  end # def run
+
+  private
+  def process_new(queue, since=nil)
+
+    if since.nil?
+        since = sincedb_read()
+    end
+
+    objects = list_new(since)
+    objects.each do |k|
+      @logger.debug("S3 input processing", :bucket => @bucket, :key => k)
+      lastmod = @s3bucket.objects[k].last_modified
+      process_log(queue, k)
+      sincedb_write(lastmod)
+    end
+
+  end # def process_new
+
+  private
+  def list_new(since=nil)
+
+    if since.nil?
+      since = Time.new(0)
+    end
+
+    objects = {}
+    @s3bucket.objects.with_prefix(@prefix).each do |log|
+      if log.last_modified > since
+        objects[log.key] = log.last_modified
+      end
+    end
+
+    return sorted_objects = objects.keys.sort {|a,b| objects[a] <=> objects[b]}
+
+  end # def list_new
+
+  private
+  def process_log(queue, key)
+
+    object = @s3bucket.objects[key]
+    tmp = Dir.mktmpdir("logstash-")
+    begin
+      filename = File.join(tmp, File.basename(key))
+      File.open(filename, 'wb') do |s3file|
+        object.read do |chunk|
+          s3file.write(chunk)
+        end
+      end
+      process_local_log(queue, filename)
+      unless @backup_to_bucket.nil?
+        backup_object = @backup_bucket.objects[key]
+        backup_object.write(Pathname.new(filename))
+      end
+      unless @backup_to_dir.nil?
+        FileUtils.cp(filename, @backup_to_dir)
+      end
+      if @delete
+        object.delete()
+      end
+    end
+    FileUtils.remove_entry_secure(tmp, force=true)
+
+  end # def process_log
+
+  private
+  def process_local_log(queue, filename)
+
+    metadata = {
+      :version => nil,
+      :format => nil,
+    }
+    File.open(filename) do |file|
+      if filename.end_with?('.gz')
+        gz = Zlib::GzipReader.new(file)
+        gz.each_line do |line|
+          metadata = process_line(queue, metadata, line)
+        end
+      else
+        file.each do |line|
+          metadata = process_line(queue, metadata, line)
+        end
+      end
+    end
+
+  end # def process_local_log
+
+  private
+  def process_line(queue, metadata, line)
+
+    if /#Version: .+/.match(line)
+      junk, version = line.strip().split(/#Version: (.+)/)
+      unless version.nil?
+        metadata[:version] = version
+      end
+    elsif /#Fields: .+/.match(line)
+      junk, format = line.strip().split(/#Fields: (.+)/)
+      unless format.nil?
+        metadata[:format] = format
+      end
+    else
+      @codec.decode(line) do |event|
+        decorate(event)
+        unless metadata[:version].nil?
+          event["cloudfront_version"] = metadata[:version]
+        end
+        unless metadata[:format].nil?
+          event["cloudfront_fields"] = metadata[:format]
+        end
+        queue << event
+      end
+    end
+    return metadata
+
+  end # def process_line
+
+  private
+  def sincedb_read()
+
+    if File.exists?(@sincedb_path)
+      since = Time.parse(File.read(@sincedb_path).chomp.strip)
+    else
+      since = Time.new(0)
+    end
+    return since
+
+  end # def sincedb_read
+
+  private
+  def sincedb_write(since=nil)
+
+    if since.nil?
+      since = Time.now()
+    end
+    File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
+
+  end # def sincedb_write
+
+end # class LogStash::Inputs::S3