logstash-output-redis 2.0.2 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 348b8b201a5c1b4037b578cd12677dcc35c7886a
-  data.tar.gz: 1d0745eba1fb41d695b31c74fa137c958cbeb31e
+  metadata.gz: a529cca11d4cde933249b32c7e76ae676d9b755b
+  data.tar.gz: 2ec4475ed8cb05a5c68e989a8e87b44394fa9016
 SHA512:
-  metadata.gz: f570f04b66c0252eac48c645c2cc0b75a827e8be4a46c42656e26a074e308d3d12bb5bf011341cc1d8a9d34e618b641fea81aef9555b90e5aac6ad890bb1e252
-  data.tar.gz: fb2498a17beaa8fe86db6aca3d11be109f8a28675dd1ac88d2ae2d06afb0a6ed0f26d99bfcb7c8b78ebd03f56802e49b263139b61fd31c39e85ac5e58a17ed4f
+  metadata.gz: 3fe3cfc505241664aa8e9eaf267090145c9cf0ec51e3099e0aa0c1e983513e53b4a6955d342a5209f0dd09f316eb050454b638f43101cfd815f366ce0bd33808
+  data.tar.gz: 39c5ec408749415ecedc7a3edb7f5f5be74aa2df9878d7323eb652dd90754f2a9ec1cdf6e7c6614234668f1540b9653472279e38df093d6892b4028236c76ec7

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.0.4
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.0.3
+  - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-redis-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-redis-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/redis.rb

@@ -18,6 +18,8 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base
 
   config_name "redis"
 
+  default :codec, "json"
+
   # Name is used for logging in case there are multiple instances.
   # TODO: delete
   config :name, :validate => :string, :default => 'default',
@@ -137,46 +139,22 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base
     @host_idx = 0
 
     @congestion_check_times = Hash.new { |h,k| h[k] = Time.now.to_i - @congestion_interval }
+
+    @codec.on_event(&method(:send_to_redis))
   end # def register
 
   def receive(event)
     
 
-    if @batch and @data_type == 'list' # Don't use batched method for pubsub.
-      # Stud::Buffer
-      buffer_receive(event.to_json, event.sprintf(@key))
-      return
-    end
-
-    key = event.sprintf(@key)
     # TODO(sissel): We really should not drop an event, but historically
     # we have dropped events that fail to be converted to json.
     # TODO(sissel): Find a way to continue passing events through even
     # if they fail to convert properly.
     begin
-      payload = event.to_json
-    rescue Encoding::UndefinedConversionError, ArgumentError
-      puts "FAILUREENCODING"
-      @logger.error("Failed to convert event to JSON. Invalid UTF-8, maybe?",
-                    :event => event.inspect)
-      return
-    end
-
-    begin
-      @redis ||= connect
-      if @data_type == 'list'
-        congestion_check(key)
-        @redis.rpush(key, payload)
-      else
-        @redis.publish(key, payload)
-      end
-    rescue => e
-      @logger.warn("Failed to send event to Redis", :event => event,
-                   :identity => identity, :exception => e,
-                   :backtrace => e.backtrace)
-      sleep @reconnect_interval
-      @redis = nil
-      retry
+      @codec.encode(event)
+    rescue StandardError => e
+      @logger.warn("Error encoding event", :exception => e,
+                   :event => event)
     end
   end # def receive
 
@@ -248,4 +226,31 @@ class LogStash::Outputs::Redis < LogStash::Outputs::Base
     @name || "redis://#{@password}@#{@current_host}:#{@current_port}/#{@db} #{@data_type}:#{@key}"
   end
 
+  def send_to_redis(event, payload)
+    # How can I do this sort of thing with codecs?
+    key = event.sprintf(@key)
+    
+    if @batch and @data_type == 'list' # Don't use batched method for pubsub.
+      # Stud::Buffer
+      buffer_receive(payload, key)
+      next
+    end
+
+    begin
+      @redis ||= connect
+      if @data_type == 'list'
+        congestion_check(key)
+        @redis.rpush(key, payload)
+      else
+        @redis.publish(key, payload)
+      end
+    rescue => e
+      @logger.warn("Failed to send event to Redis", :event => event,
+                   :identity => identity, :exception => e,
+                   :backtrace => e.backtrace)
+      sleep @reconnect_interval
+      @redis = nil
+      retry
+    end
+  end
 end

data/logstash-output-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-redis'
-  s.version         = '2.0.2'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will send events to a Redis queue using RPUSH"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,11 +20,14 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
 
   s.add_runtime_dependency 'redis'
   s.add_runtime_dependency 'stud'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'flores'
 end
 

data/spec/outputs/redis_spec.rb

@@ -2,83 +2,38 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/redis"
 require "logstash/json"
 require "redis"
+require "flores/random"
 
 describe LogStash::Outputs::Redis, :redis => true do
-  
 
-  describe "ship lots of events to a list" do
-    key = 10.times.collect { rand(10).to_s }.join("")
-    event_count = 10000 + rand(500)
-
-    config <<-CONFIG
-      input {
-        generator {
-          message => "hello world"
-          count => #{event_count}
-          type => "generator"
-        }
-      }
-      output {
-        redis {
-          host => "127.0.0.1"
-          key => "#{key}"
-          data_type => list
-        }
+  shared_examples_for "writing to redis list" do |extra_config|
+    let(:key) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { Flores::Random.integer(0..10000) }
+    let(:message) { Flores::Random.text(0..100) }
+    let(:default_config) {
+      {
+        "key" => key,
+        "data_type" => "list",
+        "host" => "localhost"
       }
-    CONFIG
-
-    agent do
-      # Query redis directly and inspect the goodness.
-      redis = Redis.new(:host => "127.0.0.1")
-
-      # The list should contain the number of elements our agent pushed up.
-      insist { redis.llen(key) } == event_count
-
-      # Now check all events for order and correctness.
-      event_count.times do |value|
-        id, element = redis.blpop(key, 0)
-        event = LogStash::Event.new(LogStash::Json.load(element))
-        insist { event["sequence"] } == value
-        insist { event["message"] } == "hello world"
+    }
+    let(:redis_config) { 
+      default_config.merge(extra_config || {})
+    }
+    let(:redis_output) { described_class.new(redis_config) }
+
+    before do
+      redis_output.register
+      event_count.times do |i|
+        event = LogStash::Event.new("sequence" => i, "message" => message)
+        redis_output.receive(event)
       end
+      redis_output.close
+    end
 
-      # The list should now be empty
-      insist { redis.llen(key) } == 0
-    end # agent
-  end
-
-  describe "batch mode" do
-    key = 10.times.collect { rand(10).to_s }.join("")
-    event_count = 200000
-
-    config <<-CONFIG
-      input {
-        generator {
-          message => "hello world"
-          count => #{event_count}
-          type => "generator"
-        }
-      }
-      output {
-        redis {
-          host => "127.0.0.1"
-          key => "#{key}"
-          data_type => list
-          batch => true
-          batch_timeout => 5
-          timeout => 5
-        }
-      }
-    CONFIG
-
-    agent do
-      # we have to wait for close to execute & flush the last batch.
-      # otherwise we might start doing assertions before everything has been
-      # sent out to redis.
-      sleep 2
-
+    it "should successfully send all events to redis" do
       redis = Redis.new(:host => "127.0.0.1")
-
+      
       # The list should contain the number of elements our agent pushed up.
       insist { redis.llen(key) } == event_count
 
@@ -87,42 +42,34 @@ describe LogStash::Outputs::Redis, :redis => true do
         id, element = redis.blpop(key, 0)
         event = LogStash::Event.new(LogStash::Json.load(element))
         insist { event["sequence"] } == value
-        insist { event["message"] } == "hello world"
+        insist { event["message"] } == message
       end
 
       # The list should now be empty
       insist { redis.llen(key) } == 0
-    end # agent
+    end
   end
 
-  describe "converts US-ASCII to utf-8 without failures" do
-    key = 10.times.collect { rand(10).to_s }.join("")
+  context "when batch_mode is false" do
+    include_examples "writing to redis list"
+  end
 
-    config <<-CONFIG
-      input {
-        generator {
-          charset => "US-ASCII"
-          message => "\xAD\u0000"
-          count => 1
-          type => "generator"
-        }
-      }
-      output {
-        redis {
-          host => "127.0.0.1"
-          key => "#{key}"
-          data_type => list
-        }
-      }
-    CONFIG
+  context "when batch_mode is true" do
+    batch_events = Flores::Random.integer(1..1000)
+    batch_settings = { 
+      "batch" => true,
+      "batch_events" => batch_events
+    }
 
-    agent do
-      # Query redis directly and inspect the goodness.
-      redis = Redis.new(:host => "127.0.0.1")
+    include_examples "writing to redis list", batch_settings do
 
-      # The list should contain no elements.
-      insist { redis.llen(key) } == 1
-    end # agent
+      # A canary to make sure we're actually enabling batch mode
+      # in this shared example.
+      it "should have batch mode enabled" do
+        expect(redis_config).to include("batch")
+        expect(redis_config["batch"]).to be_truthy
+      end
+    end
   end
 end
 

metadata

@@ -1,39 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-redis
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: redis
@@ -41,13 +35,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: stud
@@ -55,13 +49,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -69,7 +63,49 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-input-generator
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: flores
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -99,12 +135,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []