logstash-lib 1.3.2

data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb

@@ -0,0 +1 @@
+require "logstash/outputs/rabbitmq/march_hare"

data/lib/logstash/outputs/rabbitmq/march_hare.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+class LogStash::Outputs::RabbitMQ
+  module MarchHareImpl
+
+
+    #
+    # API
+    #
+
+    def register
+      require "march_hare"
+      require "java"
+
+      @logger.info("Registering output", :plugin => self)
+
+      @connected = java.util.concurrent.atomic.AtomicBoolean.new
+
+      connect
+      declare_exchange
+
+      @connected.set(true)
+
+      @codec.on_event(&method(:publish_serialized))
+    end
+
+
+    def receive(event)
+      return unless output?(event)
+
+      begin
+        @codec.encode(event)
+      rescue JSON::GeneratorError => e
+        @logger.warn("Trouble converting event to JSON", :exception => e,
+                     :event => event)
+      end
+    end
+
+    def publish_serialized(message)
+      begin
+        if @connected.get
+          @x.publish(message, :routing_key => @key, :properties => {
+            :persistent => @persistent
+          })
+        else
+          @logger.warn("Tried to send a message, but not connected to RabbitMQ.")
+        end
+      rescue MarchHare::Exception, com.rabbitmq.client.AlreadyClosedException => e
+        @connected.set(false)
+        n = 10
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+
+        connect
+        declare_exchange
+        retry
+      end
+    end
+
+    def to_s
+      return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
+    end
+
+    def teardown
+      @connected.set(false)
+      @conn.close if @conn && @conn.open?
+      @conn = nil
+
+      finished
+    end
+
+
+
+    #
+    # Implementation
+    #
+
+    def connect
+      return if terminating?
+
+      @vhost       ||= "127.0.0.1"
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= 5672
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :user  => @user,
+        :automatic_recovery => false
+      }
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                "guest"
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      proto                  = if @ssl
+                                 "amqp"
+                               else
+                                 "amqps"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      begin
+        @conn = MarchHare.connect(@settings)
+
+        @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+
+        @ch = @conn.create_channel
+        @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+      rescue MarchHare::Exception => e
+        @connected.set(false)
+        n = 10
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        retry
+      end
+    end
+
+    def declare_exchange
+      @logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
+                    :durable => @durable)
+      @x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
+
+      # sets @connected to true during recovery. MK.
+      @connected.set(true)
+
+      @x
+    end
+
+  end # MarchHareImpl
+end

data/lib/logstash/outputs/redis.rb

@@ -0,0 +1,245 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "stud/buffer"
+
+# send events to a redis database using RPUSH
+#
+# For more information about redis, see <http://redis.io/>
+class LogStash::Outputs::Redis < LogStash::Outputs::Base
+
+  include Stud::Buffer
+
+  config_name "redis"
+  milestone 2
+
+  # Name is used for logging in case there are multiple instances.
+  # TODO: delete
+  config :name, :validate => :string, :default => 'default',
+    :deprecated => true
+
+  # The hostname(s) of your redis server(s). Ports may be specified on any
+  # hostname, which will override the global port config.
+  #
+  # For example:
+  #
+  #     "127.0.0.1"
+  #     ["127.0.0.1", "127.0.0.2"]
+  #     ["127.0.0.1:6380", "127.0.0.1"]
+  config :host, :validate => :array, :default => ["127.0.0.1"]
+
+  # Shuffle the host list during logstash startup.
+  config :shuffle_hosts, :validate => :boolean, :default => true
+
+  # The default port to connect on. Can be overridden on any hostname.
+  config :port, :validate => :number, :default => 6379
+
+  # The redis database number.
+  config :db, :validate => :number, :default => 0
+
+  # Redis initial connection timeout in seconds.
+  config :timeout, :validate => :number, :default => 5
+
+  # Password to authenticate with.  There is no authentication by default.
+  config :password, :validate => :password
+
+  # The name of the redis queue (we'll use RPUSH on this). Dynamic names are
+  # valid here, for example "logstash-%{type}"
+  # TODO: delete
+  config :queue, :validate => :string, :deprecated => true
+
+  # The name of a redis list or channel. Dynamic names are
+  # valid here, for example "logstash-%{type}".
+  # TODO set required true
+  config :key, :validate => :string, :required => false
+
+  # Either list or channel.  If redis_type is list, then we will RPUSH to key.
+  # If redis_type is channel, then we will PUBLISH to key.
+  # TODO set required true
+  config :data_type, :validate => [ "list", "channel" ], :required => false
+
+  # Set to true if you want redis to batch up values and send 1 RPUSH command
+  # instead of one command per value to push on the list.  Note that this only
+  # works with data_type="list" mode right now.
+  #
+  # If true, we send an RPUSH every "batch_events" events or
+  # "batch_timeout" seconds (whichever comes first).
+  # Only supported for list redis data_type.
+  config :batch, :validate => :boolean, :default => false
+
+  # If batch is set to true, the number of events we queue up for an RPUSH.
+  config :batch_events, :validate => :number, :default => 50
+
+  # If batch is set to true, the maximum amount of time between RPUSH commands
+  # when there are pending events to flush.
+  config :batch_timeout, :validate => :number, :default => 5
+
+  # Interval for reconnecting to failed redis connections
+  config :reconnect_interval, :validate => :number, :default => 1
+
+  # In case redis data_type is list and has more than @congestion_threshold items, block until someone consumes them and reduces
+  # congestion, otherwise if there are no consumers redis will run out of memory, unless it was configured with OOM protection.
+  # But even with OOM protection single redis list can block all other users of redis, as well redis cpu consumption
+  # becomes bad then it reaches the max allowed ram size.
+  # Default value of 0 means that this limit is disabled.
+  # Only supported for list redis data_type.
+  config :congestion_threshold, :validate => :number, :default => 0
+
+  # How often to check for congestion, defaults to 1 second.
+  # Zero means to check on every event.
+  config :congestion_interval, :validate => :number, :default => 1
+
+  def register
+    require 'redis'
+
+    # TODO remove after setting key and data_type to true
+    if @queue
+      if @key or @data_type
+        raise RuntimeError.new(
+          "Cannot specify queue parameter and key or data_type"
+        )
+      end
+      @key = @queue
+      @data_type = 'list'
+    end
+
+    if not @key or not @data_type
+      raise RuntimeError.new(
+        "Must define queue, or key and data_type parameters"
+      )
+    end
+    # end TODO
+
+
+    if @batch
+      if @data_type != "list"
+        raise RuntimeError.new(
+          "batch is not supported with data_type #{@data_type}"
+        )
+      end
+      buffer_initialize(
+        :max_items => @batch_events,
+        :max_interval => @batch_timeout,
+        :logger => @logger
+      )
+    end
+
+    @redis = nil
+    if @shuffle_hosts
+        @host.shuffle!
+    end
+    @host_idx = 0
+
+    @congestion_check_times = Hash.new { |h,k| h[k] = Time.now.to_i - @congestion_interval }
+  end # def register
+
+  def receive(event)
+    return unless output?(event)
+
+    if @batch and @data_type == 'list' # Don't use batched method for pubsub.
+      # Stud::Buffer
+      buffer_receive(event.to_json, event.sprintf(@key))
+      return
+    end
+
+    key = event.sprintf(@key)
+    # TODO(sissel): We really should not drop an event, but historically
+    # we have dropped events that fail to be converted to json.
+    # TODO(sissel): Find a way to continue passing events through even
+    # if they fail to convert properly.
+    begin
+      payload = event.to_json
+    rescue Encoding::UndefinedConversionError, ArgumentError
+      puts "FAILUREENCODING"
+      @logger.error("Failed to convert event to JSON. Invalid UTF-8, maybe?",
+                    :event => event.inspect)
+      return
+    end
+
+    begin
+      @redis ||= connect
+      if @data_type == 'list'
+        congestion_check(key)
+        @redis.rpush(key, payload)
+      else
+        @redis.publish(key, payload)
+      end
+    rescue => e
+      @logger.warn("Failed to send event to redis", :event => event,
+                   :identity => identity, :exception => e,
+                   :backtrace => e.backtrace)
+      sleep @reconnect_interval
+      @redis = nil
+      retry
+    end
+  end # def receive
+
+  def congestion_check(key)
+    return if @congestion_threshold == 0
+    if (Time.now.to_i - @congestion_check_times[key]) >= @congestion_interval # Check congestion only if enough time has passed since last check.
+      while @redis.llen(key) > @congestion_threshold # Don't push event to redis key which has reached @congestion_threshold.
+        @logger.warn? and @logger.warn("Redis key size has hit a congestion threshold #{@congestion_threshold} suspending output for #{@congestion_interval} seconds")
+        sleep @congestion_interval
+      end
+      @congestion_check_time = Time.now.to_i
+    end
+  end
+
+  # called from Stud::Buffer#buffer_flush when there are events to flush
+  def flush(events, key, teardown=false)
+    @redis ||= connect
+    # we should not block due to congestion on teardown
+    # to support this Stud::Buffer#buffer_flush should pass here the :final boolean value.
+    congestion_check(key) unless teardown
+    @redis.rpush(key, events)
+  end
+  # called from Stud::Buffer#buffer_flush when an error occurs
+  def on_flush_error(e)
+    @logger.warn("Failed to send backlog of events to redis",
+      :identity => identity,
+      :exception => e,
+      :backtrace => e.backtrace
+    )
+    @redis = connect
+  end
+
+  def teardown
+    if @batch
+      buffer_flush(:final => true)
+    end
+    if @data_type == 'channel' and @redis
+      @redis.quit
+      @redis = nil
+    end
+  end
+
+  private
+  def connect
+    @current_host, @current_port = @host[@host_idx].split(':')
+    @host_idx = @host_idx + 1 >= @host.length ? 0 : @host_idx + 1
+
+    if not @current_port
+      @current_port = @port
+    end
+
+    params = {
+      :host => @current_host,
+      :port => @current_port,
+      :timeout => @timeout,
+      :db => @db
+    }
+    @logger.debug(params)
+
+    if @password
+      params[:password] = @password.value
+    end
+
+    Redis.new(params)
+  end # def connect
+
+  # A string used to identify a redis instance in log messages
+  def identity
+    @name || "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}"
+  end
+
+end

data/lib/logstash/outputs/riak.rb

@@ -0,0 +1,152 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# Riak is a distributed k/v store from Basho.
+# It's based on the Dynamo model.
+
+class LogStash::Outputs::Riak < LogStash::Outputs::Base
+  config_name "riak"
+  milestone 1
+
+  # The nodes of your Riak cluster
+  # This can be a single host or
+  # a Logstash hash of node/port pairs
+  # e.g
+  # ["node1", "8098", "node2", "8098"]
+  config :nodes, :validate => :hash, :default => {"localhost" =>  "8098"}
+
+  # The protocol to use
+  # HTTP or ProtoBuf
+  # Applies to ALL backends listed above
+  # No mix and match
+  config :proto, :validate => ["http", "pb"], :default => "http"
+
+  # The bucket name to write events to
+  # Expansion is supported here as values are 
+  # passed through event.sprintf
+  # Multiple buckets can be specified here
+  # but any bucket-specific settings defined
+  # apply to ALL the buckets.
+  config :bucket, :validate => :array, :default => ["logstash-%{+YYYY.MM.dd}"]
+
+  # The event key name
+  # variables are valid here.
+  #
+  # Choose this carefully. Best to let riak decide....
+  config :key_name, :validate => :string
+
+  # Bucket properties (NYI)
+  # Logstash hash of properties for the bucket
+  # i.e.
+  # `bucket_props => ["r", "one", "w", "one", "dw", "one"]`
+  # or
+  # `bucket_props => ["n_val", "3"]`
+  # Note that the Logstash config language cannot support
+  # hash or array values
+  # Properties will be passed as-is
+  config :bucket_props, :validate => :hash
+
+  # Indices
+  # Array of fields to add 2i on
+  # e.g.
+  # `indices => ["source_host", "type"]
+  # Off by default as not everyone runs eleveldb
+  config :indices, :validate => :array
+
+  # Search
+  # Enable search on the bucket defined above
+  config :enable_search, :validate => :boolean, :default => false
+
+  # SSL
+  # Enable SSL
+  config :enable_ssl, :validate => :boolean, :default => false
+
+  # SSL Options
+  # Options for SSL connections
+  # Only applied if SSL is enabled
+  # Logstash hash that maps to the riak-client options
+  # here: https://github.com/basho/riak-ruby-client/wiki/Connecting-to-Riak
+  # You'll likely want something like this:
+  # `ssl_opts => ["pem", "/etc/riak.pem", "ca_path", "/usr/share/certificates"]
+  # Per the riak client docs, the above sample options
+  # will turn on SSL `VERIFY_PEER`
+  config :ssl_opts, :validate => :hash
+
+  # Metadata (NYI)
+  # Allow the user to set custom metadata on the object
+  # Should consider converting logstash data to metadata as well
+  #
+
+  public
+  def register
+    require 'riak'
+    riak_opts = {}
+    cluster_nodes = Array.new
+    @logger.debug("Setting protocol", :protocol => @proto)
+    proto_type = "#{@proto}_port".to_sym
+    @nodes.each do |node,port|
+      @logger.debug("Adding node", :node => node, :port => port)
+      cluster_nodes << {:host => node, proto_type => port}
+    end
+    @logger.debug("Cluster nodes", :nodes => cluster_nodes)
+    if @enable_ssl
+      @logger.debug("SSL requested")
+      if @ssl_opts
+        @logger.debug("SSL options provided", @ssl_opts)
+        riak_opts.merge!(@ssl_opts.inject({}) {|h,(k,v)| h[k.to_sym] = v; h})
+      else
+        riak_opts.merge!({:ssl => true})
+      end
+    @logger.debug("Riak options:", :riak_opts => riak_opts)
+    end
+    riak_opts.merge!({:nodes => cluster_nodes})
+    @logger.debug("Riak options:", :riak_opts => riak_opts)
+    @client = Riak::Client.new(riak_opts)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    
+    @bucket.each do |b|
+      # setup our bucket(s)
+      bukkit = @client.bucket(event.sprintf(b))
+      # Disable bucket props for now
+      # Need to detect params passed that should be converted to int
+      # otherwise setting props fails =(
+      # Logstash syntax only supports strings and bools
+      # likely fix is to either hack in is_numeric?
+      # or whitelist certain params and call to_i
+      ##@logger.debug("Setting bucket props", :props => @bucket_props)
+      ##bukkit.props = @bucket_props if @bucket_props
+      ##@logger.debug("Bucket", :bukkit => bukkit.inspect)
+     
+      if @enable_search
+        @logger.debug("Enable search requested", :bucket => bukkit.inspect)
+        # Check if search is enabled
+        @logger.debug("Checking bucket status", :search_enabled => bukkit.is_indexed?)
+        bukkit.enable_index! unless bukkit.is_indexed?
+        @logger.debug("Rechecking bucket status", :search_enabled => bukkit.is_indexed?)
+      end
+      @key_name.nil? ? evt_key=nil : evt_key=event.sprintf(@key_name)
+      evt = Riak::RObject.new(bukkit, evt_key)
+      @logger.debug("RObject", :robject => evt.to_s)
+      begin
+        evt.content_type = "application/json"
+        evt.data = event
+        if @indices
+          @indices.each do |k|
+            idx_name = "#{k.gsub('@','')}_bin"
+            @logger.debug("Riak index name", :idx => idx_name)
+            @logger.info("Indexes", :indexes => evt.indexes.to_s)
+            evt.indexes[idx_name] << event.sprintf("%{#{k}}")
+          end
+        end
+        evt.store
+      rescue Exception => e
+        @logger.warn("Exception storing", :message => e.message)
+      end
+    end
+  end # def receive
+end # class LogStash::Outputs::Riak