logstash-input-redis 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5dc8076a13a0c8563482032f19ce6371d23befd2
-  data.tar.gz: 76d9b1bd9210129bbfe4cb3605d5ff4d19307f7a
+  metadata.gz: d9928285771d37dd118f099c40c59243cb24aedf
+  data.tar.gz: 9fa7aeb915e98e533b06aaddc91f830b2595256c
 SHA512:
-  metadata.gz: 942e3733ec6c9bda66d614eb3fa0f8ffd682c119bb055e6396024a1442e12fffb93464acdfb3ec79d8a564dc6b30ea3acabe21461dae7cc07cffae87ebfc9713
-  data.tar.gz: b286512a0bfcd241d64c44fbfd8df2eb6e5e0add94a812afa972ef1aa1d390600c02cece6022475b4a438ac39b052c732d3cebd9b92128898dc77652e75cc332
+  metadata.gz: dee7ad5a84df8293ffdb343a478caad99e1d951e2b5b344a866e25d5f19bb23128c5b63573dca91be6435142382589264f2bc9c97270454e5f7cdf853d91f1f7
+  data.tar.gz: 88deceeca56390a52fac036bba9191b15362ab6a3e972a85be4cf3ce7ea12c29d11f1a3ddc64fa98542cd05a4f4e4a9b0930ff8827253558aa6f9c670bd882cd

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 2.0.3
+ - Changed default batch size to 125. Improve batch handling code. Add travis ci build with redis integration.
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/CONTRIBUTORS

@@ -14,6 +14,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * piavlo
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,5 +1,12 @@
 # Logstash Plugin
 
+Travis Build
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-redis.svg)](https://travis-ci.org/logstash-plugins/logstash-input-redis)
+
+Jenkins Build
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-redis-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-redis-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
@@ -83,4 +90,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/redis.rb

@@ -2,6 +2,7 @@
 require "logstash/namespace"
 require "logstash/inputs/base"
 require "logstash/inputs/threadable"
+require 'redis'
 
 # This input will read events from a Redis instance; it supports both Redis channels and lists.
 # The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and
@@ -16,7 +17,7 @@ require "logstash/inputs/threadable"
 # newer. Anything older does not support the operations used by batching.
 #
 module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
-# class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
+  BATCH_EMPTY_SLEEP = 0.25
 
   config_name "redis"
 
@@ -56,7 +57,7 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   config :data_type, :validate => [ "list", "channel", "pattern_channel" ], :required => false
 
   # The number of events to return from Redis using EVAL.
-  config :batch_count, :validate => :number, :default => 1
+  config :batch_count, :validate => :number, :default => 125
 
   public
   # public API
@@ -77,7 +78,6 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   end
 
   def register
-    require 'redis'
     @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
 
     # TODO remove after setting key and data_type to true
@@ -112,6 +112,8 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
       @stop_method = method(:subscribe_stop)
     end
 
+    @list_method = batched? ? method(:list_batch_listener) : method(:list_single_listener)
+
     # TODO(sissel, boertje): set @identity directly when @name config option is removed.
     @identity = @name != 'default' ? @name : "#{@redis_url} #{@data_type}:#{@key}"
     @logger.info("Registering Redis", :identity => @identity)
@@ -165,21 +167,11 @@ module LogStash module Inputs class Redis < LogStash::Inputs::Threadable
   # private
   def load_batch_script(redis)
     #A Redis Lua EVAL script to fetch a count of keys
-    #in case count is bigger than current items in queue whole queue will be returned without extra nil values
-    redis_script = <<EOF
-          local i = tonumber(ARGV[1])
-          local res = {}
-          local length = redis.call('llen',KEYS[1])
-          if length < i then i = length end
-          while (i > 0) do
-            local item = redis.call("lpop", KEYS[1])
-            if (not item) then
-              break
-            end
-            table.insert(res, item)
-            i = i-1
-          end
-          return res
+      redis_script = <<EOF
+        local batchsize = tonumber(ARGV[1])
+        local result = redis.call('lrange', KEYS[1], 0, batchsize)
+        redis.call('ltrim', KEYS[1], batchsize + 1, -1)
+        return result
 EOF
     @redis_script_sha = redis.script(:load, redis_script)
   end
@@ -209,7 +201,7 @@ EOF
     while !stop?
       begin
         @redis ||= connect
-        list_listener(@redis, output_queue)
+        @list_method.call(@redis, output_queue)
       rescue ::Redis::BaseError => e
         @logger.warn("Redis connection problem", :exception => e)
         # Reset the redis variable to trigger reconnect
@@ -221,24 +213,17 @@ EOF
     end
   end
 
-  # private
-  def list_listener(redis, output_queue)
-
-    item = redis.blpop(@key, 0, :timeout => 1)
-    return unless item # from timeout or other conditions
-
-    # blpop returns the 'key' read from as well as the item result
-    # we only care about the result (2nd item in the list).
-    queue_event(item.last, output_queue)
-
-    # If @batch_count is 1, there's no need to continue.
-    return if !batched?
-
+  def list_batch_listener(redis, output_queue)
     begin
-      redis.evalsha(@redis_script_sha, [@key], [@batch_count-1]).each do |item|
+      results = redis.evalsha(@redis_script_sha, [@key], [@batch_count-1])
+      results.each do |item|
         queue_event(item, output_queue)
       end
 
+      if results.size.zero?
+        sleep BATCH_EMPTY_SLEEP
+      end
+
       # Below is a commented-out implementation of 'batch fetch'
       # using pipelined LPOP calls. This in practice has been observed to
       # perform exactly the same in terms of event throughput as
@@ -253,6 +238,8 @@ EOF
         #queue_event(item, output_queue) if item
       #end
       # --- End commented out implementation of 'batch fetch'
+      # further to the above, the LUA script now uses lrange and trim
+      # which should further improve the efficiency of the script
     rescue ::Redis::CommandError => e
       if e.to_s =~ /NOSCRIPT/ then
         @logger.warn("Redis may have been restarted, reloading Redis batch EVAL script", :exception => e);
@@ -264,6 +251,15 @@ EOF
     end
   end
 
+  def list_single_listener(redis, output_queue)
+    item = redis.blpop(@key, 0, :timeout => 1)
+    return unless item # from timeout or other conditions
+
+    # blpop returns the 'key' read from as well as the item result
+    # we only care about the result (2nd item in the list).
+    queue_event(item.last, output_queue)
+  end
+
   # private
   def subscribe_stop
     return if @redis.nil? || !@redis.connected?

data/logstash-input-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-redis'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This input will read events from a Redis instance"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/redis_spec.rb

@@ -2,6 +2,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "redis"
 require "stud/try"
 require 'logstash/inputs/redis'
+require 'securerandom'
 
 def populate(key, event_count)
   require "logstash/event"
@@ -19,17 +20,15 @@ def process(conf, event_count)
     event_count.times.map{queue.pop}
   end
 
-  events.each_with_index do |event, i|
-    insist { event["sequence"] } == i
-  end
-end # process
+  expect(events.map{|evt| evt["sequence"]}).to eq((0..event_count.pred).to_a)
+end
 
 # integration tests ---------------------
 
 describe "inputs/redis", :redis => true do
 
   it "should read events from a list" do
-    key = 10.times.collect { rand(10).to_s }.join("")
+    key = SecureRandom.hex
     event_count = 1000 + rand(50)
     # event_count = 100
     conf = <<-CONFIG
@@ -38,6 +37,7 @@ describe "inputs/redis", :redis => true do
           type => "blah"
           key => "#{key}"
           data_type => "list"
+          batch_count => 1
         }
       }
     CONFIG
@@ -46,8 +46,8 @@ describe "inputs/redis", :redis => true do
     process(conf, event_count)
   end
 
-  it "should read events from a list using batch_count" do
-    key = 10.times.collect { rand(10).to_s }.join("")
+  it "should read events from a list using batch_count (default 125)" do
+    key = SecureRandom.hex
     event_count = 1000 + rand(50)
     conf = <<-CONFIG
       input {
@@ -55,7 +55,6 @@ describe "inputs/redis", :redis => true do
           type => "blah"
           key => "#{key}"
           data_type => "list"
-          batch_count => #{rand(20)+1}
         }
       }
     CONFIG
@@ -73,7 +72,8 @@ describe LogStash::Inputs::Redis do
   let(:connection) { double('redis_connection') }
   let(:connected) { [true] }
   let(:data_type) { 'list' }
-  let(:cfg) { {'key' => 'foo', 'data_type' => data_type} }
+  let(:batch_count) { 1 }
+  let(:cfg) { {'key' => 'foo', 'data_type' => data_type, 'batch_count' => batch_count} }
   let(:quit_calls) { [:quit] }
   let(:accumulator) { [] }
 
@@ -123,6 +123,65 @@ describe LogStash::Inputs::Redis do
       expect(accumulator.size).to be > 0
     end
 
+    context "when the batch size is greater than 1" do
+      let(:batch_count) { 10 }
+      let(:rates) { [] }
+
+      before do
+        allow(redis).to receive(:connected?).and_return(connected.last)
+        allow(redis).to receive(:script)
+        allow(redis).to receive(:quit)
+      end
+
+      it 'calling the run method, adds events to the queue' do
+        expect(redis).to receive(:evalsha).at_least(:once).and_return(['a', 'b'])
+
+        tt = Thread.new do
+          sleep 0.01
+          subject.do_stop
+        end
+
+        subject.run(accumulator)
+
+        tt.join
+        expect(accumulator.size).to be > 0
+      end
+    end
+
+    context "when there is no data" do
+      let(:batch_count) { 10 }
+      let(:rates) { [] }
+
+      it 'will throttle the loop' do
+        allow(redis).to receive(:evalsha) do
+          rates.unshift Time.now.to_f
+          []
+        end
+        allow(redis).to receive(:connected?).and_return(connected.last)
+        allow(redis).to receive(:script)
+        allow(redis).to receive(:quit)
+
+        tt = Thread.new do
+          sleep 1
+          subject.do_stop
+        end
+
+        subject.run(accumulator)
+
+        tt.join
+
+        inters = []
+        rates.each_cons(2) do |x, y|
+          inters << x - y
+        end
+
+        expect(accumulator.size).to eq(0)
+        inters.each do |delta|
+          expect(delta).to be_within(0.01).of(LogStash::Inputs::Redis::BATCH_EMPTY_SLEEP)
+        end
+      end
+    end
+
     it 'multiple close calls, calls to redis once' do
       subject.use_redis(redis)
       allow(redis).to receive(:blpop).and_return(['foo', 'l1'])

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-redis
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,48 +28,50 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: redis
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: redis
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []