logstash-output-redis 3.0.3 → 3.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c65f7320d9ce46d8e81d66028de147d14022271
-  data.tar.gz: ac8a08c2e5d8815fd0ea972be411b66d549728b4
+  metadata.gz: 219f28da639de092f4c59fbeee8e429dd1d0e5d1
+  data.tar.gz: a72a78f8012bf3546ef9b339b8f0772f81f0840d
 SHA512:
-  metadata.gz: 3d0ae828a25891aee0a63cb35fb2ee6caffa0e9bf93a53f463fb3cdd62ca6b7555c4b89ff1065b77dc7dd25c8eeb881064e50194131a9cfb5fb006119b388793
-  data.tar.gz: fca979637bb9dfb15beec69d7cd8cc43be871a593741248716bf53c939befc0550cd3815c9ad796483cee742b613cd59749f1ceafd66557c3f307aaa2509f5b1
+  metadata.gz: 5206e62bcbab82013a79144b700eee53fc0806ab3a209774e5ce7276a2031bf647037cc59e3fa7b59549954a844495286a0339f24a98ccd3e975f698cff9d511
+  data.tar.gz: 234c53b89d0c0457bb1a190e093a05982116b9ee47b2ffad345793e536c6c71e9ebd15d180de11eaa342c4009ccdc643c00c4ebd70493e928cf6f52eec8d76e8

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,221 @@
+:plugin: redis
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Redis output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This output will send events to a Redis queue using RPUSH.
+The RPUSH command is supported in Redis v0.0.7+. Using
+PUBLISH to a channel requires at least v1.3.8+.
+While you may be able to make these Redis versions work,
+the best performance and stability will be found in more
+recent stable versions.  Versions 2.6.0+ are recommended.
+
+For more information, see http://redis.io/[the Redis homepage]
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Redis Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-batch>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-batch_events>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-batch_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-congestion_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-congestion_threshold>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-data_type>> |<<string,string>>, one of `["list", "channel"]`|No
+| <<plugins-{type}s-{plugin}-db>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-host>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-shuffle_hosts>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-batch"]
+===== `batch` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Set to true if you want Redis to batch up values and send 1 RPUSH command
+instead of one command per value to push on the list.  Note that this only
+works with `data_type="list"` mode right now.
+
+If true, we send an RPUSH every "batch_events" events or
+"batch_timeout" seconds (whichever comes first).
+Only supported for `data_type` is "list".
+
+[id="plugins-{type}s-{plugin}-batch_events"]
+===== `batch_events` 
+
+  * Value type is <<number,number>>
+  * Default value is `50`
+
+If batch is set to true, the number of events we queue up for an RPUSH.
+
+[id="plugins-{type}s-{plugin}-batch_timeout"]
+===== `batch_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+If batch is set to true, the maximum amount of time between RPUSH commands
+when there are pending events to flush.
+
+[id="plugins-{type}s-{plugin}-congestion_interval"]
+===== `congestion_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+How often to check for congestion. Default is one second.
+Zero means to check on every event.
+
+[id="plugins-{type}s-{plugin}-congestion_threshold"]
+===== `congestion_threshold` 
+
+  * Value type is <<number,number>>
+  * Default value is `0`
+
+In case Redis `data_type` is `list` and has more than `@congestion_threshold` items,
+block until someone consumes them and reduces congestion, otherwise if there are
+no consumers Redis will run out of memory, unless it was configured with OOM protection.
+But even with OOM protection, a single Redis list can block all other users of Redis,
+until Redis CPU consumption reaches the max allowed RAM size.
+A default value of 0 means that this limit is disabled.
+Only supported for `list` Redis `data_type`.
+
+[id="plugins-{type}s-{plugin}-data_type"]
+===== `data_type` 
+
+  * Value can be any of: `list`, `channel`
+  * There is no default value for this setting.
+
+Either list or channel.  If `redis_type` is list, then we will set
+RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`.
+
+[id="plugins-{type}s-{plugin}-db"]
+===== `db` 
+
+  * Value type is <<number,number>>
+  * Default value is `0`
+
+The Redis database number.
+
+[id="plugins-{type}s-{plugin}-host"]
+===== `host` 
+
+  * Value type is <<array,array>>
+  * Default value is `["127.0.0.1"]`
+
+The hostname(s) of your Redis server(s). Ports may be specified on any
+hostname, which will override the global port config.
+If the hosts list is an array, Logstash will pick one random host to connect to,
+if that host is disconnected it will then pick another.
+
+For example:
+[source,ruby]
+    "127.0.0.1"
+    ["127.0.0.1", "127.0.0.2"]
+    ["127.0.0.1:6380", "127.0.0.1"]
+
+[id="plugins-{type}s-{plugin}-key"]
+===== `key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of a Redis list or channel. Dynamic names are
+valid here, for example `logstash-%{type}`.
+
+[id="plugins-{type}s-{plugin}-name"]
+===== `name`  (DEPRECATED)
+
+  * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
+  * Value type is <<string,string>>
+  * Default value is `"default"`
+
+Name is used for logging in case there are multiple instances.
+
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Password to authenticate with.  There is no authentication by default.
+
+[id="plugins-{type}s-{plugin}-port"]
+===== `port` 
+
+  * Value type is <<number,number>>
+  * Default value is `6379`
+
+The default port to connect on. Can be overridden on any hostname.
+
+[id="plugins-{type}s-{plugin}-queue"]
+===== `queue`  (DEPRECATED)
+
+  * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the Redis queue (we'll use RPUSH on this). Dynamic names are
+valid here, for example `logstash-%{type}`
+
+[id="plugins-{type}s-{plugin}-reconnect_interval"]
+===== `reconnect_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Interval for reconnecting to failed Redis connections
+
+[id="plugins-{type}s-{plugin}-shuffle_hosts"]
+===== `shuffle_hosts` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Shuffle the host list during Logstash startup.
+
+[id="plugins-{type}s-{plugin}-timeout"]
+===== `timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+Redis initial connection timeout in seconds.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-output-redis.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-redis'
-  s.version         = '3.0.3'
+  s.version         = '3.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will send events to a Redis queue using RPUSH"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/unit/outputs/redis_spec.rb

@@ -11,14 +11,15 @@ describe LogStash::Outputs::Redis do
     # TODO: refactor specs above and probably rely on a Redis mock to correctly test the code expected behaviour, the actual
     # tests agains Redis should be moved into integration tests.
     let(:key) { "thekey" }
-    let(:payload) { "somepayload"}
-    let(:event) { LogStash::Event.new({"message" => "test"}) }
     let(:config) {
       {
         "key" => key,
         "data_type" => "list",
         "batch" => true,
         "batch_events" => 50,
+        "batch_timeout" => 3600 * 24,
+        # ^ this a very large timeout value to prevent the Flush Timer thread in Stud::Buffer from calling flush
+        # it screws with the RSpec expect().to receive thread safety.
        }
     }
     let(:redis) { described_class.new(config) }
@@ -27,12 +28,13 @@ describe LogStash::Outputs::Redis do
       redis.register
       expect(redis).to receive(:buffer_receive).exactly(10000).times.and_call_original
       expect(redis).to receive(:flush).exactly(200).times
+      expect(redis).not_to receive(:on_flush_error)
 
       # I was able to reproduce the LocalJumpError: unexpected next exception at around 50
       # consicutive invocations. setting to 10000 should reproduce it for any environment
       # I have no clue at this point why this problem does not happen at every invocation
-      1.upto(10000) do
-        expect{redis.receive(event)}.to_not raise_error
+      10000.times do |i|
+        expect{redis.receive(LogStash::Event.new({"message" => "test-#{i}"}))}.to_not raise_error
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-redis
 version: !ruby/object:Gem::Version
-  version: 3.0.3
+  version: 3.0.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/redis.rb
 - logstash-output-redis.gemspec
 - spec/integration/outputs/redis_spec.rb