logstash-input-redis 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f3c4e21e1c031b454b041be5fd084352b6439dea
-  data.tar.gz: 9116bd121eea1697bacfdab2a90e5b08fbe13488
+  metadata.gz: ddb2a94b6e3c3cff06790e2434bc9f6aaad46760
+  data.tar.gz: e251dbde7ef57f43b921c6ebf2d6cd3d897d1889
 SHA512:
-  metadata.gz: 3bb19842af185d1a2fb32f41b6f90b620827c509741dcd5fdb4c1fe092a10d04d01431c8718f6b98b215c24fe506a4658252bc161d14bb3e19971f2f9e36513a
-  data.tar.gz: ea05bcbb2ce3b47c21663970817ecd3c7e88c4fd9e21a341d96f22e2556f0b3336e3d00e4af349082b3dda0b9579b3cc5a5a070c55f6eb8913608b486a6c7dc6
+  metadata.gz: ea5587b7e19d151161b5065a1efaff804d62621e58c919feac72b01c6f3d7e5e2915422f18e3afcbe721ecf8a6dec69252a92b0fa215aab2c7f9b614502345d6
+  data.tar.gz: bdf90d25cef6ac38a917251428a057c7007f6226058985a18f97d58509412689c4eb630ea328ce469c6b07b1c7d9a84849133448dd807de22415529893ac7af2

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.1
+ - Make teardown more reliable
+ - Re-organise code and tests

data/lib/logstash/inputs/redis.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
+require "logstash/namespace"
 require "logstash/inputs/base"
 require "logstash/inputs/threadable"
-require "logstash/namespace"
 
 # This input will read events from a Redis instance; it supports both Redis channels and lists.
 # The list command (BLPOP) used by Logstash is supported in Redis v1.3.1+, and
@@ -15,7 +15,9 @@ require "logstash/namespace"
 # `batch_count` note: If you use the `batch_count` setting, you *must* use a Redis version 2.6.0 or
 # newer. Anything older does not support the operations used by batching.
 #
-class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
+module Logstash module Inputs class Redis < LogStash::Inputs::Threadable
+# class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
+
   config_name "redis"
 
   default :codec, "json"
@@ -57,14 +59,30 @@ class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
   config :batch_count, :validate => :number, :default => 1
 
   public
+  # public API
+  # use to store a proc that can provide a redis instance or mock
+  def add_external_redis_builder(builder) #callable
+    @redis_builder = builder
+    self
+  end
+
+  # use to apply an instance directly and bypass the builder
+  def use_redis(instance)
+    @redis = instance
+    self
+  end
+
+  def new_redis_instance
+    @redis_builder.call
+  end
+
   def register
     require 'redis'
-    @redis = nil
     @redis_url = "redis://#{@password}@#{@host}:#{@port}/#{@db}"
 
     # TODO remove after setting key and data_type to true
     if @queue
-      if @key or @data_type
+      if @key || @data_type
         raise RuntimeError.new(
           "Cannot specify queue parameter and key or data_type"
         )
@@ -73,38 +91,79 @@ class LogStash::Inputs::Redis < LogStash::Inputs::Threadable
       @data_type = 'list'
     end
 
-    if not @key or not @data_type
+    if !@key || !@data_type
       raise RuntimeError.new(
         "Must define queue, or key and data_type parameters"
       )
     end
     # end TODO
 
-    @logger.info("Registering Redis", :identity => identity)
+    @redis_builder ||= method(:internal_redis_builder)
+
+    # just switch on data_type once
+    if @data_type == 'list' || @data_type == 'dummy'
+      @run_method = method(:list_runner)
+      @teardown_method = method(:list_teardown)
+    elsif @data_type == 'channel'
+      @run_method = method(:channel_runner)
+      @teardown_method = method(:subscribe_teardown)
+    elsif @data_type == 'pattern_channel'
+      @run_method = method(:pattern_channel_runner)
+      @teardown_method = method(:subscribe_teardown)
+    end
+
+    # TODO(sissel, boertje): set @identity directly when @name config option is removed.
+    @identity = @name != 'default' ? @name : "#{@redis_url} #{@data_type}:#{@key}"
+    @logger.info("Registering Redis", :identity => @identity)
   end # def register
 
-  # A string used to identify a Redis instance in log messages
-  # TODO(sissel): Use instance variables for this once the @name config
-  # option is removed.
-  private
-  def identity
-    @name || "#{@redis_url} #{@data_type}:#{@key}"
+  def run(output_queue)
+    @run_method.call(output_queue)
+  rescue LogStash::ShutdownSignal
+    # ignore and quit
+  end # def run
+
+  def teardown
+    @shutdown_requested = true
+    @teardown_method.call
   end
 
+  # private methods -----------------------------
   private
-  def connect
-    redis = Redis.new(
+
+  def batched?
+    @batch_count > 1
+  end
+
+  # private
+  def is_list_type?
+    @data_type == 'list'
+  end
+
+  # private
+  def redis_params
+    {
       :host => @host,
       :port => @port,
       :timeout => @timeout,
       :db => @db,
       :password => @password.nil? ? nil : @password.value
-    )
-    load_batch_script(redis) if @data_type == 'list' && (@batch_count > 1)
-    return redis
+    }
+  end
+
+  # private
+  def internal_redis_builder
+    ::Redis.new(redis_params)
+  end
+
+  # private
+  def connect
+    redis = new_redis_instance
+    load_batch_script(redis) if batched? && is_list_type?
+    redis
   end # def connect
 
-  private
+  # private
   def load_batch_script(redis)
     #A Redis Lua EVAL script to fetch a count of keys
     #in case count is bigger than current items in queue whole queue will be returned without extra nil values
@@ -126,7 +185,7 @@ EOF
     @redis_script_sha = redis.script(:load, redis_script)
   end
 
-  private
+  # private
   def queue_event(msg, output_queue)
     begin
       @codec.decode(msg) do |event|
@@ -141,7 +200,40 @@ EOF
     end
   end
 
-  private
+  # private
+  def shutting_down?
+    @shutdown_requested
+  end
+
+  # private
+  def running?
+    !@shutdown_requested
+  end
+
+  # private
+  def list_teardown
+    return if @redis.nil? || !@redis.connected?
+
+    @redis.quit rescue nil
+    @redis = nil
+  end
+
+  # private
+  def list_runner(output_queue)
+    while running?
+      begin
+        @redis ||= connect
+        list_listener(@redis, output_queue)
+      rescue ::Redis::BaseError => e
+        @logger.warn("Redis connection problem", :exception => e)
+        # Reset the redis variable to trigger reconnect
+        @redis = nil
+        sleep 1
+      end
+    end
+  end
+
+  # private
   def list_listener(redis, output_queue)
 
     item = redis.blpop(@key, 0, :timeout => 1)
@@ -149,10 +241,10 @@ EOF
 
     # blpop returns the 'key' read from as well as the item result
     # we only care about the result (2nd item in the list).
-    queue_event(item[1], output_queue)
+    queue_event(item.last, output_queue)
 
     # If @batch_count is 1, there's no need to continue.
-    return if @batch_count == 1
+    return if !batched?
 
     begin
       redis.evalsha(@redis_script_sha, [@key], [@batch_count-1]).each do |item|
@@ -173,7 +265,7 @@ EOF
         #queue_event(item, output_queue) if item
       #end
       # --- End commented out implementation of 'batch fetch'
-    rescue Redis::CommandError => e
+    rescue ::Redis::CommandError => e
       if e.to_s =~ /NOSCRIPT/ then
         @logger.warn("Redis may have been restarted, reloading Redis batch EVAL script", :exception => e);
         load_batch_script(redis)
@@ -184,15 +276,49 @@ EOF
     end
   end
 
-  private
-  def channel_listener(redis, output_queue)
-    redis.subscribe @key do |on|
+  # private
+  def subscribe_teardown
+    return if @redis.nil? || !@redis.connected?
+    # if its a SubscribedClient then:
+    # it does not have a disconnect method (yet)
+    if @redis.client.is_a?(::Redis::SubscribedClient)
+      @redis.client.unsubscribe
+    else
+      @redis.client.disconnect
+    end
+    @redis = nil
+  end
+
+  # private
+  def redis_runner
+    begin
+      @redis ||= connect
+      yield
+    rescue ::Redis::BaseError => e
+      @logger.warn("Redis connection problem", :exception => e)
+      # Reset the redis variable to trigger reconnect
+      @redis = nil
+      sleep 1
+      retry
+    end
+  end
+
+  # private
+  def channel_runner(output_queue)
+    redis_runner do
+      channel_listener(output_queue)
+    end
+  end
+
+  # private
+  def channel_listener(output_queue)
+    @redis.subscribe(@key) do |on|
       on.subscribe do |channel, count|
         @logger.info("Subscribed", :channel => channel, :count => count)
       end
 
       on.message do |channel, message|
-        queue_event message, output_queue
+        queue_event(message, output_queue)
       end
 
       on.unsubscribe do |channel, count|
@@ -201,15 +327,21 @@ EOF
     end
   end
 
-  private
-  def pattern_channel_listener(redis, output_queue)
-    redis.psubscribe @key do |on|
+  def pattern_channel_runner(output_queue)
+    redis_runner do
+      pattern_channel_listener(output_queue)
+    end
+  end
+
+  # private
+  def pattern_channel_listener(output_queue)
+    @redis.psubscribe @key do |on|
       on.psubscribe do |channel, count|
         @logger.info("Subscribed", :channel => channel, :count => count)
       end
 
-      on.pmessage do |ch, event, message|
-        queue_event message, output_queue
+      on.pmessage do |pattern, channel, message|
+        queue_event(message, output_queue)
       end
 
       on.punsubscribe do |channel, count|
@@ -218,51 +350,6 @@ EOF
     end
   end
 
-  # Since both listeners have the same basic loop, we've abstracted the outer
-  # loop.
-  private
-  def listener_loop(listener, output_queue)
-    while !@shutdown_requested
-      begin
-        @redis ||= connect
-        self.send listener, @redis, output_queue
-      rescue Redis::BaseError => e
-        @logger.warn("Redis connection problem", :exception => e)
-        # Reset the redis variable to trigger reconnect
-        @redis = nil
-        sleep 1
-      end
-    end
-  end # listener_loop
-
-  public
-  def run(output_queue)
-    if @data_type == 'list'
-      listener_loop :list_listener, output_queue
-    elsif @data_type == 'channel'
-      listener_loop :channel_listener, output_queue
-    else
-      listener_loop :pattern_channel_listener, output_queue
-    end
-  rescue LogStash::ShutdownSignal
-    # ignore and quit
-  end # def run
-
-  public
-  def teardown
-    @shutdown_requested = true
+# end
 
-    if @redis
-      if @data_type == 'list'
-        @redis.quit rescue nil
-      elsif @data_type == 'channel'
-        @redis.unsubscribe rescue nil
-        @redis.connection.disconnect
-      elsif @data_type == 'pattern_channel'
-        @redis.punsubscribe rescue nil
-        @redis.connection.disconnect
-      end
-      @redis = nil
-    end
-  end
-end # class LogStash::Inputs::Redis
+end end end # Redis Inputs  LogStash

data/logstash-input-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-redis'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This input will read events from a Redis instance"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/redis_spec.rb

@@ -1,6 +1,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "redis"
 require "stud/try"
+require 'logstash/inputs/redis'
 
 def populate(key, event_count)
   require "logstash/event"
@@ -23,6 +24,8 @@ def process(conf, event_count)
   end
 end # process
 
+# integration tests ---------------------
+
 describe "inputs/redis", :redis => true do
 
   it "should read events from a list" do
@@ -61,3 +64,178 @@ describe "inputs/redis", :redis => true do
     process(conf, event_count)
   end
 end
+
+# unit tests ---------------------
+
+describe Logstash::Inputs::Redis do
+  let(:redis) { double('redis') }
+  let(:builder) { ->{ redis } }
+  let(:connection) { double('redis_connection') }
+  let(:connected) { [true] }
+  let(:data_type) { 'list' }
+  let(:cfg) { {'key' => 'foo', 'data_type' => data_type} }
+  let(:quit_calls) { [:quit] }
+  let(:accumulator) { [] }
+
+  subject do
+    described_class.new(cfg).add_external_redis_builder(builder)
+  end
+
+  context 'construction' do
+    it 'registers the input' do
+      expect {subject.register}.not_to raise_error
+    end
+  end
+
+  context 'runtime for list data_type' do
+    before do
+      subject.register
+    end
+
+    context 'teardown when redis is unset' do
+      let(:quit_calls) { [:quit, :unsubscribe, :punsubscribe, :connection, :disconnect!] }
+
+      it 'does not attempt to quit' do
+        allow(redis).to receive(:nil?).and_return(true)
+        quit_calls.each do |call|
+          expect(redis).not_to receive(call)
+        end
+        expect {subject.teardown}.not_to raise_error
+      end
+    end
+
+    it 'calling the run method, adds events to the queue' do
+      expect(redis).to receive(:blpop).at_least(:once).and_return(['foo', 'l1'])
+
+      allow(redis).to receive(:connected?).and_return(connected.last)
+      allow(redis).to receive(:quit)
+
+      tt = Thread.new do
+        sleep 0.01
+        subject.teardown
+      end
+
+      subject.run(accumulator)
+
+      tt.join
+
+      expect(accumulator.size).to be > 0
+    end
+
+    it 'multiple teardown calls, calls to redis once' do
+      subject.use_redis(redis)
+      allow(redis).to receive(:blpop).and_return(['foo', 'l1'])
+      expect(redis).to receive(:connected?).and_return(connected.last)
+      quit_calls.each do |call|
+        expect(redis).to receive(call).at_most(:once)
+      end
+
+      subject.teardown
+      connected.push(false) #can't use let block here so push to array
+      expect {subject.teardown}.not_to raise_error
+      subject.teardown
+    end
+  end
+
+  context 'for the subscribe data_types' do
+    def run_it_thread(inst)
+      Thread.new(inst) do |subj|
+        subj.run(accumulator)
+      end
+    end
+
+    def publish_thread(new_redis, prefix)
+      Thread.new(new_redis, prefix) do |r, p|
+        sleep 0.1
+        2.times do |i|
+          r.publish('foo', "#{p}#{i.next}")
+        end
+      end
+    end
+
+    def teardown_thread(inst, rt)
+      Thread.new(inst, rt) do |subj, runner|
+        sleep 0.4 # allow the messages through
+        runner.raise(LogStash::ShutdownSignal)
+        subj.teardown
+      end
+    end
+
+    let(:instance) do
+      inst = described_class.new(cfg)
+      inst.register
+      inst
+    end
+
+    before do
+      subject.register
+      subject.use_redis(redis)
+      allow(connection).to receive(:is_a?).and_return(true)
+      allow(redis).to receive(:client).and_return(connection)
+    end
+
+    before(:example, type: :mocked) do
+      expect(redis).to receive(:connected?).and_return(connected.last)
+      expect(connection).to receive(:unsubscribe)
+
+      quit_calls.each do |call|
+        expect(redis).to receive(call).at_most(:once)
+      end
+    end
+
+    context 'runtime for channel data_type' do
+      let(:data_type) { 'channel' }
+      let(:quit_calls) { [:unsubscribe, :connection] }
+
+      context 'mocked redis' do
+        it 'multiple teardown calls, calls to redis once', type: :mocked do
+          subject.teardown
+          connected.push(false) #can't use let block here so push to array
+          expect {subject.teardown}.not_to raise_error
+          subject.teardown
+        end
+      end
+
+      context 'real redis', :redis => true do
+        it 'calling the run method, adds events to the queue' do
+          #simulate the input thread
+          rt = run_it_thread(instance)
+          #simulate the other system thread
+          publish_thread(instance.new_redis_instance, 'c').join
+          #simulate the pipeline thread
+          teardown_thread(instance, rt).join
+
+          expect(accumulator.size).to eq(2)
+        end
+      end
+    end
+
+    context 'runtime for pattern_channel data_type' do
+      let(:data_type)  { 'pattern_channel' }
+      let(:quit_calls) { [:punsubscribe, :connection] }
+
+      context 'mocked redis' do
+        it 'multiple teardown calls, calls to redis once', type: :mocked do
+          subject.teardown
+          connected.push(false) #can't use let block here so push to array
+          expect {subject.teardown}.not_to raise_error
+          subject.teardown
+        end
+      end
+
+      context 'real redis', :redis => true do
+        it 'calling the run method, adds events to the queue' do
+          #simulate the input thread
+          rt = run_it_thread(instance)
+          #simulate the other system thread
+          publish_thread(instance.new_redis_instance, 'pc').join
+          #simulate the pipeline thread
+          teardown_thread(instance, rt).join
+
+          expect(accumulator.size).to eq(2)
+        end
+      end
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-redis
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -111,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: This input will read events from a Redis instance