logstash-output-redis 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a529cca11d4cde933249b32c7e76ae676d9b755b
-  data.tar.gz: 2ec4475ed8cb05a5c68e989a8e87b44394fa9016
+  metadata.gz: e5521e16c30f64515f0280b3dac67304d69704a0
+  data.tar.gz: 0cdffaddc9e1d3347be833351cb6a31981061174
 SHA512:
-  metadata.gz: 3fe3cfc505241664aa8e9eaf267090145c9cf0ec51e3099e0aa0c1e983513e53b4a6955d342a5209f0dd09f316eb050454b638f43101cfd815f366ce0bd33808
-  data.tar.gz: 39c5ec408749415ecedc7a3edb7f5f5be74aa2df9878d7323eb652dd90754f2a9ec1cdf6e7c6614234668f1540b9653472279e38df093d6892b4028236c76ec7
+  metadata.gz: 7e489069572645b18d6db062d0cf4712ac4087d92fa88b5c54b9f78485407085bd18bc4cca54a15de908ed6657b5a37596dce2f11fa5805acb7de2a64d4882ec
+  data.tar.gz: b398711520f6e4667f5c18a487c6efeea300058152c6b31bf7b659bbd85a5ec7c6ac6c0e90feba3ec0c1024191579ffd067c7b2053c5b516e54e4b96f10fdac8

data/CHANGELOG.md

@@ -1,9 +1,11 @@
+# 2.0.5
+  - Fix LocalJumpError exception see https://github.com/logstash-plugins/logstash-output-redis/pull/27
 # 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-redis-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-redis-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-redis.svg)](https://travis-ci.org/logstash-plugins/logstash-output-redis)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/lib/logstash/outputs/redis.rb

@@ -7,7 +7,7 @@ require "stud/buffer"
 # The RPUSH command is supported in Redis v0.0.7+. Using
 # PUBLISH to a channel requires at least v1.3.8+.
 # While you may be able to make these Redis versions work,
-# the best performance and stability will be found in more 
+# the best performance and stability will be found in more
 # recent stable versions.  Versions 2.6.0+ are recommended.
 #
 # For more information, see http://redis.io/[the Redis homepage]
@@ -144,14 +144,17 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base
   end # def register
 
   def receive(event)
-    
-
     # TODO(sissel): We really should not drop an event, but historically
     # we have dropped events that fail to be converted to json.
     # TODO(sissel): Find a way to continue passing events through even
     # if they fail to convert properly.
     begin
       @codec.encode(event)
+    rescue LocalJumpError
+      # This LocalJumpError rescue clause is required to test for regressions
+      # for https://github.com/logstash-plugins/logstash-output-redis/issues/26
+      # see specs. Without it the LocalJumpError is rescued by the StandardError
+      raise
     rescue StandardError => e
       @logger.warn("Error encoding event", :exception => e,
                    :event => event)
@@ -229,11 +232,11 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base
   def send_to_redis(event, payload)
     # How can I do this sort of thing with codecs?
     key = event.sprintf(@key)
-    
-    if @batch and @data_type == 'list' # Don't use batched method for pubsub.
+
+    if @batch && @data_type == 'list' # Don't use batched method for pubsub.
       # Stud::Buffer
       buffer_receive(payload, key)
-      next
+      return
     end
 
     begin

data/logstash-output-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-redis'
-  s.version         = '2.0.4'
+  s.version         = '2.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will send events to a Redis queue using RPUSH"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/redis_spec.rb

@@ -4,70 +4,103 @@ require "logstash/json"
 require "redis"
 require "flores/random"
 
-describe LogStash::Outputs::Redis, :redis => true do
+describe LogStash::Outputs::Redis do
 
-  shared_examples_for "writing to redis list" do |extra_config|
-    let(:key) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:event_count) { Flores::Random.integer(0..10000) }
-    let(:message) { Flores::Random.text(0..100) }
-    let(:default_config) {
-      {
-        "key" => key,
-        "data_type" => "list",
-        "host" => "localhost"
+  context "integration tests", :redis => true do
+    shared_examples_for "writing to redis list" do |extra_config|
+      let(:key) { 10.times.collect { rand(10).to_s }.join("") }
+      let(:event_count) { Flores::Random.integer(0..10000) }
+      let(:message) { Flores::Random.text(0..100) }
+      let(:default_config) {
+        {
+          "key" => key,
+          "data_type" => "list",
+          "host" => "localhost"
+        }
       }
-    }
-    let(:redis_config) { 
-      default_config.merge(extra_config || {})
-    }
-    let(:redis_output) { described_class.new(redis_config) }
+      let(:redis_config) {
+        default_config.merge(extra_config || {})
+      }
+      let(:redis_output) { described_class.new(redis_config) }
 
-    before do
-      redis_output.register
-      event_count.times do |i|
-        event = LogStash::Event.new("sequence" => i, "message" => message)
-        redis_output.receive(event)
+      before do
+        redis_output.register
+        event_count.times do |i|
+          event = LogStash::Event.new("sequence" => i, "message" => message)
+          redis_output.receive(event)
+        end
+        redis_output.close
       end
-      redis_output.close
-    end
 
-    it "should successfully send all events to redis" do
-      redis = Redis.new(:host => "127.0.0.1")
-      
-      # The list should contain the number of elements our agent pushed up.
-      insist { redis.llen(key) } == event_count
+      it "should successfully send all events to redis" do
+        redis = Redis.new(:host => "127.0.0.1")
+
+        # The list should contain the number of elements our agent pushed up.
+        insist { redis.llen(key) } == event_count
 
-      # Now check all events for order and correctness.
-      event_count.times do |value|
-        id, element = redis.blpop(key, 0)
-        event = LogStash::Event.new(LogStash::Json.load(element))
-        insist { event["sequence"] } == value
-        insist { event["message"] } == message
+        # Now check all events for order and correctness.
+        event_count.times do |value|
+          id, element = redis.blpop(key, 0)
+          event = LogStash::Event.new(LogStash::Json.load(element))
+          insist { event["sequence"] } == value
+          insist { event["message"] } == message
+        end
+
+        # The list should now be empty
+        insist { redis.llen(key) } == 0
       end
+    end
 
-      # The list should now be empty
-      insist { redis.llen(key) } == 0
+    context "when batch_mode is false" do
+      include_examples "writing to redis list"
     end
-  end
 
-  context "when batch_mode is false" do
-    include_examples "writing to redis list"
+    context "when batch_mode is true" do
+      batch_events = Flores::Random.integer(1..1000)
+      batch_settings = {
+        "batch" => true,
+        "batch_events" => batch_events
+      }
+
+      include_examples "writing to redis list", batch_settings do
+
+        # A canary to make sure we're actually enabling batch mode
+        # in this shared example.
+        it "should have batch mode enabled" do
+          expect(redis_config).to include("batch")
+          expect(redis_config["batch"]).to be_truthy
+        end
+      end
+    end
   end
 
-  context "when batch_mode is true" do
-    batch_events = Flores::Random.integer(1..1000)
-    batch_settings = { 
-      "batch" => true,
-      "batch_events" => batch_events
+  context "Redis#receive in batch mode" do
+    # this is a regression test harness to verify fix for https://github.com/logstash-plugins/logstash-output-redis/issues/26
+    # TODO: refactor specs above and probably rely on a Redis mock to correctly test the code expected behaviour, the actual
+    # tests agains Redis should be moved into integration tests.
+    let(:key) { "thekey" }
+    let(:payload) { "somepayload"}
+    let(:event) { LogStash::Event.new({"message" => "test"}) }
+    let(:config) {
+      {
+        "key" => key,
+        "data_type" => "list",
+        "batch" => true,
+        "batch_events" => 50,
+       }
     }
+    let(:redis) { described_class.new(config) }
 
-    include_examples "writing to redis list", batch_settings do
+    it "should call buffer_receive" do
+      redis.register
+      expect(redis).to receive(:buffer_receive).exactly(10000).times.and_call_original
+      expect(redis).to receive(:flush).exactly(200).times
 
-      # A canary to make sure we're actually enabling batch mode
-      # in this shared example.
-      it "should have batch mode enabled" do
-        expect(redis_config).to include("batch")
-        expect(redis_config["batch"]).to be_truthy
+      # I was able to reproduce the LocalJumpError: unexpected next exception at around 50
+      # consicutive invocations. setting to 10000 should reproduce it for any environment
+      # I have no clue at this point why this problem does not happen at every invocation
+      1.upto(10000) do
+        expect{redis.receive(event)}.to_not raise_error
       end
     end
   end

metadata

@@ -1,113 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-redis
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: redis
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: redis
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: stud
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-input-generator
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-input-generator
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: flores
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -135,12 +135,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []