logstash-input-redis-cluster 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35a910295d21a98489ddf116a6ab82e6278f0781
-  data.tar.gz: d658953749222517f3be6e0875b9a64c36ad8400
+  metadata.gz: a3c3e95623f3ffc9e04ce65dabc3b23a8d375963
+  data.tar.gz: e82a979f444c1ea185f9ee9d95e7a208022bb368
 SHA512:
-  metadata.gz: 21100167f1dae24db67e4bec8c510d5686a65afd773e84d02345eeb675b78a0e2f0e970c3e1d6e92b14491e435607cfc013e5bf6631be8b40f2c03c06ae99ffb
-  data.tar.gz: b59b5f6e5c669c3ad31ace64f6335c8575c49f54fb603ca20e4ef0559b4142286a505aa43fb4e7a3ea1f7bb756843ce0e14784762b5469f096124f35b0e7c868
+  metadata.gz: c2cf58a65112ce42b6840ff4afbacbc444abef37074506a038942091c3ae739265e71f6c1c2532a3f1e7f92660700dce93509c06e021ea334368ff78db810448
+  data.tar.gz: 1a577020960613781b7146493e4fd26e1f80d009e872f13752697d20dbece3b65ed70cd1b9bb84ce4d43db891c7aae694c3cacfab2617bc182392946cfcb9455

data/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 gemspec
-gem "redis-rb-cluster", :git => "git://github.com/Tschef/redis-rb-cluster.git"
+gem "redis-rb-cluster", :git => "git://github.com/Tschef/redis-rb-cluster.git"
+gem "hiredis", "~> 0.4.0"

data/lib/logstash/inputs/redis_cluster.rb

@@ -1,330 +1,364 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/inputs/base"
-require "logstash/inputs/threadable"
-
-# This input will read events from a Redis instance; it supports both Redis channels and lists.
-# The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and
-# the channel commands used by Logstash are found in Redis v1.3.8+.
-# While you may be able to make these Redis versions work, the best performance
-# and stability will be found in more recent stable versions.  Versions 2.6.0+
-# are recommended.
-#
-# For more information about Redis, see <http://redis.io/>
-#
-# `batch_count` note: If you use the `batch_count` setting, you *must* use a Redis version 2.6.0 or
-# newer. Anything older does not support the operations used by batching.
-#
-module LogStash module Inputs class RedisCluster < LogStash::Inputs::Threadable
-# class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
-
-  config_name "redis_cluster"
-
-  default :codec, "json"
-
-  # The `name` configuration is used for logging in case there are multiple instances.
-  # This feature has no real function and will be removed in future versions.
-  config :name, :validate => :string, :default => "default", :deprecated => true
-
-  # The hostname of your Redis server.
-  config :host, :validate => :string, :default => "127.0.0.1"
-
-  # The port to connect on.
-  config :port, :validate => :number, :default => 6379
-
-  # The Redis database number.
-  config :db, :validate => :number, :default => 0
-
-  # Initial connection timeout in seconds.
-  config :timeout, :validate => :number, :default => 5
-
-  # Password to authenticate with. There is no authentication by default.
-  config :password, :validate => :password
-
-  # The name of a Redis list or channel.
-  # TODO: change required to true
-  config :keys, :validate => :array
-
-  # Specify either list or channel.  If `redis\_type` is `list`, then we will BLPOP the
-  # key.  If `redis\_type` is `channel`, then we will SUBSCRIBE to the key.
-  # If `redis\_type` is `pattern_channel`, then we will PSUBSCRIBE to the key.
-  # TODO: change required to true
-  config :data_type, :validate => [ "list", "channel", "pattern_channel" ], :required => false
-
-  # The number of events to return from Redis using EVAL.
-  config :batch_count, :validate => :number, :default => 1
-
-  public
-  # public API
-  # use to store a proc that can provide a redis instance or mock
-  def add_external_redis_builder(builder) #callable
-    @redis_builder = builder
-    self
-  end
-
-  # use to apply an instance directly and bypass the builder
-  def use_redis(instance)
-    @redis = instance
-    self
-  end
-
-  def new_redis_instance
-    @redis_builder.call
-  end
-
-  def register
-    require 'redis-rb-cluster'
-    @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
-
-    # TODO remove after setting key and data_type to true
-   
-    if !@keys || !@data_type
-      raise RuntimeError.new(
-        "Must define queue, or key and data_type parameters"
-      )
-    end
-    # end TODO
-
-    @redis_builder ||= method(:internal_redis_builder)
-
-    # just switch on data_type once
-    if @data_type == 'list' || @data_type == 'dummy'
-      @run_method = method(:list_runner)
-      @stop_method = method(:list_stop)
-    elsif @data_type == 'channel'
-      @run_method = method(:channel_runner)
-      @stop_method = method(:subscribe_stop)
-    elsif @data_type == 'pattern_channel'
-      @run_method = method(:pattern_channel_runner)
-      @stop_method = method(:subscribe_stop)
-    end
-
-    # TODO(sissel, boertje): set @identity directly when @name config option is removed.
-    @identity = @name != 'default' ? @name : "#{@redis_url} #{@data_type}"
-    @logger.info("Registering Redis", :identity => @identity)
-  end # def register
-
-  def run(output_queue)
-    @run_method.call(output_queue)
-  rescue LogStash::ShutdownSignal
-    # ignore and quit
-  end # def run
-
-  def stop
-    @stop_method.call
-  end
-
-  # private methods -----------------------------
-  private
-
-  def batched?
-    @batch_count > 1
-  end
-
-  # private
-  def is_list_type?
-    @data_type == 'list'
-  end
-
-  # private
-  def redis_params
-    {
-      :host => @host,
-      :port => @port,
-      :timeout => @timeout,
-      :db => @db,
-      :password => @password.nil? ? nil : @password.value
-    }
-  end
-
-  # private
-  def internal_redis_builder
-    ::RedisCluster.new([redis_params],4)
-  end
-
-  # private
-  def connect
-    redis = new_redis_instance
-    load_batch_script(redis) if batched? && is_list_type?
-    redis
-  end # def connect
-
-  # private
-  def load_batch_script(redis)
-    #A Redis Lua EVAL script to fetch a count of keys
-    #in case count is bigger than current items in queue whole queue will be returned without extra nil values
-    redis_script = <<EOF
-          local i = tonumber(ARGV[1])
-          local res = {}
-          local length = redis.call('llen',KEYS[1])
-          if length < i then i = length end
-          while (i > 0) do
-            local item = redis.call("lpop", KEYS[1])
-            if (not item) then
-              break
-            end
-            table.insert(res, item)
-            i = i-1
-          end
-          return res
-EOF
-    @redis_script_sha = redis.script(:load, redis_script)
-  end
-
-  # private
-  def queue_event(msg, output_queue)
-    begin
-      @codec.decode(msg) do |event|
-        decorate(event)
-        output_queue << event
-      end
-    rescue => e # parse or event creation error
-      @logger.error("Failed to create event", :message => msg, :exception => e, :backtrace => e.backtrace);
-    end
-  end
-
-  # private
-  def list_stop
-    return if @redis.nil? || !@redis.connected?
-
-    @redis.quit rescue nil
-    @redis = nil
-  end
-
-  # private
-  def list_runner(output_queue)
-    while !stop?
-      begin
-        @redis ||= connect
-        list_listener(@redis, output_queue)
-      rescue ::Redis::BaseError => e
-        @logger.warn("Redis connection problem", :exception => e)
-        # Reset the redis variable to trigger reconnect
-        @redis = nil
-        # this sleep does not need to be stoppable as its
-        # in a while !stop? loop
-        sleep 1
-      end
-    end
-  end
-
-  # private
-  def list_listener(redis, output_queue)
-	sampled = @keys.sample
-    item = redis.blpop(sampled, :timeout => 1)
-    return unless item # from timeout or other conditions
-
-    # blpop returns the 'key' read from as well as the item result
-    # we only care about the result (2nd item in the list).
-    queue_event(item.last, output_queue)
-
-    # If @batch_count is 1, there's no need to continue.
-    return if !batched?
-
-    begin
-      redis.evalsha(@redis_script_sha, [sampled], [@batch_count-1]).each do |item|
-        queue_event(item, output_queue)
-      end
-
-      # Below is a commented-out implementation of 'batch fetch'
-      # using pipelined LPOP calls. This in practice has been observed to
-      # perform exactly the same in terms of event throughput as
-      # the evalsha method. Given that the EVALSHA implementation uses
-      # one call to Redis instead of N (where N == @batch_count) calls,
-      # I decided to go with the 'evalsha' method of fetching N items
-      # from Redis in bulk.
-      #redis.pipelined do
-        #error, item = redis.lpop(@key)
-        #(@batch_count-1).times { redis.lpop(@key) }
-      #end.each do |item|
-        #queue_event(item, output_queue) if item
-      #end
-      # --- End commented out implementation of 'batch fetch'
-    rescue ::Redis::CommandError => e
-      if e.to_s =~ /NOSCRIPT/ then
-        @logger.warn("Redis may have been restarted, reloading Redis batch EVAL script", :exception => e);
-        load_batch_script(redis)
-        retry
-      else
-        raise e
-      end
-    end
-  end
-
-  # private
-  def subscribe_stop
-    return if @redis.nil? || !@redis.connected?
-    # if its a SubscribedClient then:
-    # it does not have a disconnect method (yet)
-    if @redis.client.is_a?(::Redis::SubscribedClient)
-      @redis.client.unsubscribe
-    else
-      @redis.client.disconnect
-    end
-    @redis = nil
-  end
-
-  # private
-  def redis_runner
-    begin
-      @redis ||= connect
-      yield
-    rescue ::Redis::BaseError => e
-      @logger.warn("Redis connection problem", :exception => e)
-      # Reset the redis variable to trigger reconnect
-      @redis = nil
-      Stud.stoppable_sleep(1) { stop? }
-      retry if !stop?
-    end
-  end
-
-  # private
-  def channel_runner(output_queue)
-    redis_runner do
-      channel_listener(output_queue)
-    end
-  end
-
-  # private
-  def channel_listener(output_queue)
-    @redis.subscribe(@keys.sample) do |on|
-      on.subscribe do |channel, count|
-        @logger.info("Subscribed", :channel => channel, :count => count)
-      end
-
-      on.message do |channel, message|
-        queue_event(message, output_queue)
-      end
-
-      on.unsubscribe do |channel, count|
-        @logger.info("Unsubscribed", :channel => channel, :count => count)
-      end
-    end
-  end
-
-  def pattern_channel_runner(output_queue)
-    redis_runner do
-      pattern_channel_listener(output_queue)
-    end
-  end
-
-  # private
-  def pattern_channel_listener(output_queue)
-    @redis.psubscribe @keys.sample do |on|
-      on.psubscribe do |channel, count|
-        @logger.info("Subscribed", :channel => channel, :count => count)
-      end
-
-      on.pmessage do |pattern, channel, message|
-        queue_event(message, output_queue)
-      end
-
-      on.punsubscribe do |channel, count|
-        @logger.info("Unsubscribed", :channel => channel, :count => count)
-      end
-    end
-  end
-
-# end
-
-end end end # Redis Inputs  LogStash
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/inputs/base"
+require "logstash/inputs/threadable"
+
+# This input will read events from a Redis instance; it supports both Redis channels and lists.
+# The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and
+# the channel commands used by Logstash are found in Redis v1.3.8+.
+# While you may be able to make these Redis versions work, the best performance
+# and stability will be found in more recent stable versions.  Versions 2.6.0+
+# are recommended.
+#
+# For more information about Redis, see <http://redis.io/>
+#
+# `batch_count` note: If you use the `batch_count` setting, you *must* use a Redis version 2.6.0 or
+# newer. Anything older does not support the operations used by batching.
+#
+module LogStash module Inputs class RedisCluster < LogStash::Inputs::Threadable
+# class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
+
+  config_name "redis_cluster"
+
+  default :codec, "json"
+
+  # The `name` configuration is used for logging in case there are multiple instances.
+  # This feature has no real function and will be removed in future versions.
+  config :name, :validate => :string, :default => "default", :deprecated => true
+
+  # The hostname of your Redis server.
+  config :host, :validate => :string, :default => "127.0.0.1"
+  
+  # The hostname of your Redis server.
+  config :driver, :validate => :string, :default => "jedis"
+
+  # The port to connect on.
+  config :port, :validate => :number, :default => 6379
+
+  # The Redis database number.
+  config :db, :validate => :number, :default => 0
+
+  # Initial connection timeout in seconds.
+  config :timeout, :validate => :number, :default => 5
+
+  # Initial connection timeout in seconds.
+  config :max_connections, :validate => :number, :default => 256
+
+  # Password to authenticate with. There is no authentication by default.
+  config :password, :validate => :password
+
+  # The name of a Redis list or channel.
+  # TODO: change required to true
+  config :keys, :validate => :array
+
+  # Specify either list or channel.  If `redis\_type` is `list`, then we will BLPOP the
+  # key.  If `redis\_type` is `channel`, then we will SUBSCRIBE to the key.
+  # If `redis\_type` is `pattern_channel`, then we will PSUBSCRIBE to the key.
+  # TODO: change required to true
+  config :data_type, :validate => [ "list", "channel", "pattern_channel" ], :required => false
+
+  # The number of events to return from Redis using EVAL.
+  config :batch_count, :validate => :number, :default => 1
+
+  public
+  # public API
+  # use to store a proc that can provide a redis instance or mock
+  def add_external_redis_builder(builder) #callable
+    @redis_builder = builder
+    self
+  end
+
+  # use to apply an instance directly and bypass the builder
+  def use_redis(instance)
+    @redis = instance
+    self
+  end
+
+  def new_redis_instance
+    @redis_builder.call
+  end
+
+  def register
+    @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
+
+	@batch_offset = Random.rand(1024)
+    # TODO remove after setting key and data_type to true
+	
+	
+	if @driver == "jedis" then
+		require 'org/apache/commons/commons-pool2/2.3/commons-pool2-2.3.jar'
+		require 'redis/clients/jedis/2.7.2/jedis-2.7.2.jar'
+		@error_handler = method(:error_handler_jedis)
+	else
+		require 'org/apache/commons/commons-pool2/2.3/redis-rb-cluster'
+		@error_handler = method(:error_handler_redis)
+	end
+   
+    if !@keys || !@data_type
+      raise RuntimeError.new(
+        "Must define queue, or key and data_type parameters"
+      )
+    end
+    # end TODO
+
+    @redis_builder ||= method(:internal_redis_builder)
+
+    # just switch on data_type once
+    if @data_type == 'list' || @data_type == 'dummy'
+      @run_method = method(:list_runner)
+      @stop_method = method(:list_stop)
+    elsif @data_type == 'channel'
+      @run_method = method(:channel_runner)
+      @stop_method = method(:subscribe_stop)
+    elsif @data_type == 'pattern_channel'
+      @run_method = method(:pattern_channel_runner)
+      @stop_method = method(:subscribe_stop)
+    end
+
+    # TODO(sissel, boertje): set @identity directly when @name config option is removed.
+    @identity = @name != 'default' ? @name : "#{@redis_url} #{@data_type}"
+    @logger.info("Registering Redis", :identity => @identity)
+  end # def register
+
+  def run(output_queue)
+    @run_method.call(output_queue)
+  rescue LogStash::ShutdownSignal
+    # ignore and quit
+  end # def run
+
+  def stop
+    @stop_method.call
+  end
+
+  # private methods -----------------------------
+  private
+
+  def batched?
+    @batch_count > 1
+  end
+
+  # private
+  def is_list_type?
+    @data_type == 'list'
+  end
+
+  # private
+  def redis_params
+    {
+      :host => @host,
+      :port => @port,
+      :db => @db,
+      :password => @password.nil? ? nil : @password.value,
+	  :driver => @driver,
+	  :timeout => @timeout
+    }
+  end
+
+  # private
+  def internal_redis_builder
+	if @driver == "jedis" then
+		import "redis.clients.jedis.JedisCluster"
+		import "redis.clients.jedis.HostAndPort"
+		::JedisCluster.new(java.util.HashSet.new([HostAndPort.new(redis_params[:host],redis_params[:port])]))
+	else
+		::RedisCluster.new([redis_params], @max_connections)
+	end
+  end
+
+  # private
+  def connect
+    redis = new_redis_instance
+    redis
+  end # def connect
+
+  # private
+  def load_batch_script(redis)
+    #A Redis Lua EVAL script to fetch a count of keys
+    #in case count is bigger than current items in queue whole queue will be returned without extra nil values
+    redis_script = <<EOF
+          local i = tonumber(ARGV[1])
+          local res = {}
+          local length = redis.call('llen',KEYS[1])
+          if length < i then i = length end
+          while (i > 0) do
+            local item = redis.call("lpop", KEYS[1])
+            if (not item) then
+              break
+            end
+            table.insert(res, item)
+            i = i-1
+          end
+          return res
+EOF
+    @redis_script_sha = redis.script(:load, redis_script)
+  end
+
+  # private
+  def queue_event(msg, output_queue)
+    begin
+      @codec.decode(msg) do |event|
+        decorate(event)
+        output_queue << event
+      end
+    rescue => e # parse or event creation error
+      @logger.error("Failed to create event", :message => msg, :exception => e, :backtrace => e.backtrace);
+    end
+  end
+
+  # private
+  def list_stop
+    return if @redis.nil? || !@redis.connected?
+
+    @redis.quit rescue nil
+    @redis = nil
+  end
+  
+  # private
+  def error_handler_redis(func)
+	begin
+        func.call
+      rescue ::Redis::BaseError => e
+        @logger.warn("Redis connection problem", :exception => e)
+        # Reset the redis variable to trigger reconnect
+        @redis = nil
+        
+		return false
+      end
+	  
+	  return true
+  end
+  
+  def error_handler_jedis(func)
+	import "redis.clients.jedis.exceptions.JedisException"
+	begin
+        func.call
+	  rescue ::JedisException => e
+		@logger.warn("Redis connection problem", :exception => e)
+		# Reset the redis variable to trigger reconnect
+		@redis = nil
+		
+		return false
+	  end
+	  
+	  return true
+  end
+
+  # private
+  def list_runner(output_queue)
+    while !stop?
+		if !@error_handler.call(lambda {
+			@redis ||= connect
+			list_listener(@redis, output_queue)
+		}) then
+			sleep 1
+		end
+    end
+  end
+
+  # private
+  def list_listener(redis, output_queue)
+	if batched? then	
+		for i in 1..@batch_count do
+			item = redis.lpop(@keys[(i+@batch_offset)%(@keys.length)])
+			queue_event(item, output_queue) if item
+		end
+		@batch_offset += 1
+	end
+  
+	sampled = @keys.sample
+	if @driver == "jedis" then
+		item = redis.blpop(1, sampled)
+	else
+		item = redis.blpop(sampled, :timeout => 1)
+	end
+    return unless item # from timeout or other conditions
+
+    # blpop returns the 'key' read from as well as the item result
+    # we only care about the result (2nd item in the list).
+    if @driver == "jedis" then
+        item = item.get(item.size()-1);
+    else
+        item = item.last
+    end
+    queue_event(item, output_queue)
+  end
+
+  # private
+  def subscribe_stop
+    return if @redis.nil? || !@redis.connected?
+    # if its a SubscribedClient then:
+    # it does not have a disconnect method (yet)
+	if @driver == "jedis" then
+	  @redis.quit
+    elsif @redis.client.is_a?(::Redis::SubscribedClient)
+      @redis.client.unsubscribe
+    else
+      @redis.client.disconnect
+    end
+    @redis = nil
+  end
+
+  # private
+  def redis_runner
+	if @error_handler.call(lambda {
+      @redis ||= connect
+	}) then
+      yield
+	else
+      # Reset the redis variable to trigger reconnect
+      @redis = nil
+      Stud.stoppable_sleep(1) { stop? }
+      retry if !stop?
+	end
+  end
+
+  # private
+  def channel_runner(output_queue)
+    redis_runner do
+      channel_listener(output_queue)
+    end
+  end
+
+  # private
+  def channel_listener(output_queue)
+    @redis.subscribe(@keys.sample) do |on|
+      on.subscribe do |channel, count|
+        @logger.info("Subscribed", :channel => channel, :count => count)
+      end
+
+      on.message do |channel, message|
+        queue_event(message, output_queue)
+      end
+
+      on.unsubscribe do |channel, count|
+        @logger.info("Unsubscribed", :channel => channel, :count => count)
+      end
+    end
+  end
+
+  def pattern_channel_runner(output_queue)
+    redis_runner do
+      pattern_channel_listener(output_queue)
+    end
+  end
+
+  # private
+  def pattern_channel_listener(output_queue)
+    @redis.psubscribe @keys.sample do |on|
+      on.psubscribe do |channel, count|
+        @logger.info("Subscribed", :channel => channel, :count => count)
+      end
+
+      on.pmessage do |pattern, channel, message|
+        queue_event(message, output_queue)
+      end
+
+      on.punsubscribe do |channel, count|
+        @logger.info("Unsubscribed", :channel => channel, :count => count)
+      end
+    end
+  end
+
+# end
+
+end end end # Redis Inputs  LogStash

data/logstash-input-redis-cluster.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-redis-cluster'
-  s.version         = '2.0.3'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This input will read events from a Redis instance"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
 
   s.add_runtime_dependency 'logstash-codec-json'
+  s.add_runtime_dependency "hiredis", "~> 0.4.0"
   s.add_runtime_dependency 'redis-rb-cluster'
 
   s.add_development_dependency 'logstash-devutils'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-redis-cluster
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-28 00:00:00.000000000 Z
+date: 2015-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -44,6 +44,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  name: hiredis
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -84,7 +98,10 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- lib/jedis-2.7.2.jar
 - lib/logstash/inputs/redis_cluster.rb
+- lib/org/apache/commons/commons-pool2/2.3/commons-pool2-2.3.jar
+- lib/redis/clients/jedis/2.7.2/jedis-2.7.2.jar
 - logstash-input-redis-cluster.gemspec
 - spec/inputs/redis_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html