logstash-output-redis 3.0.3 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2c65f7320d9ce46d8e81d66028de147d14022271
4
- data.tar.gz: ac8a08c2e5d8815fd0ea972be411b66d549728b4
3
+ metadata.gz: 219f28da639de092f4c59fbeee8e429dd1d0e5d1
4
+ data.tar.gz: a72a78f8012bf3546ef9b339b8f0772f81f0840d
5
5
  SHA512:
6
- metadata.gz: 3d0ae828a25891aee0a63cb35fb2ee6caffa0e9bf93a53f463fb3cdd62ca6b7555c4b89ff1065b77dc7dd25c8eeb881064e50194131a9cfb5fb006119b388793
7
- data.tar.gz: fca979637bb9dfb15beec69d7cd8cc43be871a593741248716bf53c939befc0550cd3815c9ad796483cee742b613cd59749f1ceafd66557c3f307aaa2509f5b1
6
+ metadata.gz: 5206e62bcbab82013a79144b700eee53fc0806ab3a209774e5ce7276a2031bf647037cc59e3fa7b59549954a844495286a0339f24a98ccd3e975f698cff9d511
7
+ data.tar.gz: 234c53b89d0c0457bb1a190e093a05982116b9ee47b2ffad345793e536c6c71e9ebd15d180de11eaa342c4009ccdc643c00c4ebd70493e928cf6f52eec8d76e8
data/Gemfile CHANGED
@@ -1,4 +1,11 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
- # Specify your gem's dependencies in logstash-mass_effect.gemspec
4
3
  gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,221 @@
1
+ :plugin: redis
2
+ :type: output
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Redis output plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ This output will send events to a Redis queue using RPUSH.
24
+ The RPUSH command is supported in Redis v0.0.7+. Using
25
+ PUBLISH to a channel requires at least v1.3.8+.
26
+ While you may be able to make these Redis versions work,
27
+ the best performance and stability will be found in more
28
+ recent stable versions. Versions 2.6.0+ are recommended.
29
+
30
+ For more information, see http://redis.io/[the Redis homepage]
31
+
32
+
33
+ [id="plugins-{type}s-{plugin}-options"]
34
+ ==== Redis Output Configuration Options
35
+
36
+ This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
37
+
38
+ [cols="<,<,<",options="header",]
39
+ |=======================================================================
40
+ |Setting |Input type|Required
41
+ | <<plugins-{type}s-{plugin}-batch>> |<<boolean,boolean>>|No
42
+ | <<plugins-{type}s-{plugin}-batch_events>> |<<number,number>>|No
43
+ | <<plugins-{type}s-{plugin}-batch_timeout>> |<<number,number>>|No
44
+ | <<plugins-{type}s-{plugin}-congestion_interval>> |<<number,number>>|No
45
+ | <<plugins-{type}s-{plugin}-congestion_threshold>> |<<number,number>>|No
46
+ | <<plugins-{type}s-{plugin}-data_type>> |<<string,string>>, one of `["list", "channel"]`|No
47
+ | <<plugins-{type}s-{plugin}-db>> |<<number,number>>|No
48
+ | <<plugins-{type}s-{plugin}-host>> |<<array,array>>|No
49
+ | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
50
+ | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
51
+ | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
52
+ | <<plugins-{type}s-{plugin}-reconnect_interval>> |<<number,number>>|No
53
+ | <<plugins-{type}s-{plugin}-shuffle_hosts>> |<<boolean,boolean>>|No
54
+ | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
55
+ |=======================================================================
56
+
57
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
58
+ output plugins.
59
+
60
+ &nbsp;
61
+
62
+ [id="plugins-{type}s-{plugin}-batch"]
63
+ ===== `batch`
64
+
65
+ * Value type is <<boolean,boolean>>
66
+ * Default value is `false`
67
+
68
+ Set to true if you want Redis to batch up values and send 1 RPUSH command
69
+ instead of one command per value to push on the list. Note that this only
70
+ works with `data_type="list"` mode right now.
71
+
72
+ If true, we send an RPUSH every "batch_events" events or
73
+ "batch_timeout" seconds (whichever comes first).
74
+ Only supported for `data_type` is "list".
75
+
76
+ [id="plugins-{type}s-{plugin}-batch_events"]
77
+ ===== `batch_events`
78
+
79
+ * Value type is <<number,number>>
80
+ * Default value is `50`
81
+
82
+ If batch is set to true, the number of events we queue up for an RPUSH.
83
+
84
+ [id="plugins-{type}s-{plugin}-batch_timeout"]
85
+ ===== `batch_timeout`
86
+
87
+ * Value type is <<number,number>>
88
+ * Default value is `5`
89
+
90
+ If batch is set to true, the maximum amount of time between RPUSH commands
91
+ when there are pending events to flush.
92
+
93
+ [id="plugins-{type}s-{plugin}-congestion_interval"]
94
+ ===== `congestion_interval`
95
+
96
+ * Value type is <<number,number>>
97
+ * Default value is `1`
98
+
99
+ How often to check for congestion. Default is one second.
100
+ Zero means to check on every event.
101
+
102
+ [id="plugins-{type}s-{plugin}-congestion_threshold"]
103
+ ===== `congestion_threshold`
104
+
105
+ * Value type is <<number,number>>
106
+ * Default value is `0`
107
+
108
+ In case Redis `data_type` is `list` and has more than `@congestion_threshold` items,
109
+ block until someone consumes them and reduces congestion, otherwise if there are
110
+ no consumers Redis will run out of memory, unless it was configured with OOM protection.
111
+ But even with OOM protection, a single Redis list can block all other users of Redis,
112
+ until Redis CPU consumption reaches the max allowed RAM size.
113
+ A default value of 0 means that this limit is disabled.
114
+ Only supported for `list` Redis `data_type`.
115
+
116
+ [id="plugins-{type}s-{plugin}-data_type"]
117
+ ===== `data_type`
118
+
119
+ * Value can be any of: `list`, `channel`
120
+ * There is no default value for this setting.
121
+
122
+ Either list or channel. If `redis_type` is list, then we will set
123
+ RPUSH to key. If `redis_type` is channel, then we will PUBLISH to `key`.
124
+
125
+ [id="plugins-{type}s-{plugin}-db"]
126
+ ===== `db`
127
+
128
+ * Value type is <<number,number>>
129
+ * Default value is `0`
130
+
131
+ The Redis database number.
132
+
133
+ [id="plugins-{type}s-{plugin}-host"]
134
+ ===== `host`
135
+
136
+ * Value type is <<array,array>>
137
+ * Default value is `["127.0.0.1"]`
138
+
139
+ The hostname(s) of your Redis server(s). Ports may be specified on any
140
+ hostname, which will override the global port config.
141
+ If the hosts list is an array, Logstash will pick one random host to connect to,
142
+ if that host is disconnected it will then pick another.
143
+
144
+ For example:
145
+ [source,ruby]
146
+ "127.0.0.1"
147
+ ["127.0.0.1", "127.0.0.2"]
148
+ ["127.0.0.1:6380", "127.0.0.1"]
149
+
150
+ [id="plugins-{type}s-{plugin}-key"]
151
+ ===== `key`
152
+
153
+ * Value type is <<string,string>>
154
+ * There is no default value for this setting.
155
+
156
+ The name of a Redis list or channel. Dynamic names are
157
+ valid here, for example `logstash-%{type}`.
158
+
159
+ [id="plugins-{type}s-{plugin}-name"]
160
+ ===== `name` (DEPRECATED)
161
+
162
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
163
+ * Value type is <<string,string>>
164
+ * Default value is `"default"`
165
+
166
+ Name is used for logging in case there are multiple instances.
167
+
168
+ [id="plugins-{type}s-{plugin}-password"]
169
+ ===== `password`
170
+
171
+ * Value type is <<password,password>>
172
+ * There is no default value for this setting.
173
+
174
+ Password to authenticate with. There is no authentication by default.
175
+
176
+ [id="plugins-{type}s-{plugin}-port"]
177
+ ===== `port`
178
+
179
+ * Value type is <<number,number>>
180
+ * Default value is `6379`
181
+
182
+ The default port to connect on. Can be overridden on any hostname.
183
+
184
+ [id="plugins-{type}s-{plugin}-queue"]
185
+ ===== `queue` (DEPRECATED)
186
+
187
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
188
+ * Value type is <<string,string>>
189
+ * There is no default value for this setting.
190
+
191
+ The name of the Redis queue (we'll use RPUSH on this). Dynamic names are
192
+ valid here, for example `logstash-%{type}`
193
+
194
+ [id="plugins-{type}s-{plugin}-reconnect_interval"]
195
+ ===== `reconnect_interval`
196
+
197
+ * Value type is <<number,number>>
198
+ * Default value is `1`
199
+
200
+ Interval for reconnecting to failed Redis connections
201
+
202
+ [id="plugins-{type}s-{plugin}-shuffle_hosts"]
203
+ ===== `shuffle_hosts`
204
+
205
+ * Value type is <<boolean,boolean>>
206
+ * Default value is `true`
207
+
208
+ Shuffle the host list during Logstash startup.
209
+
210
+ [id="plugins-{type}s-{plugin}-timeout"]
211
+ ===== `timeout`
212
+
213
+ * Value type is <<number,number>>
214
+ * Default value is `5`
215
+
216
+ Redis initial connection timeout in seconds.
217
+
218
+
219
+
220
+ [id="plugins-{type}s-{plugin}-common-options"]
221
+ include::{include_path}/{type}.asciidoc[]
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-redis'
4
- s.version = '3.0.3'
4
+ s.version = '3.0.4'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This output will send events to a Redis queue using RPUSH"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
14
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
15
15
 
16
16
  # Tests
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -11,14 +11,15 @@ describe LogStash::Outputs::Redis do
11
11
  # TODO: refactor specs above and probably rely on a Redis mock to correctly test the code expected behaviour, the actual
12
12
  # tests agains Redis should be moved into integration tests.
13
13
  let(:key) { "thekey" }
14
- let(:payload) { "somepayload"}
15
- let(:event) { LogStash::Event.new({"message" => "test"}) }
16
14
  let(:config) {
17
15
  {
18
16
  "key" => key,
19
17
  "data_type" => "list",
20
18
  "batch" => true,
21
19
  "batch_events" => 50,
20
+ "batch_timeout" => 3600 * 24,
21
+ # ^ this a very large timeout value to prevent the Flush Timer thread in Stud::Buffer from calling flush
22
+ # it screws with the RSpec expect().to receive thread safety.
22
23
  }
23
24
  }
24
25
  let(:redis) { described_class.new(config) }
@@ -27,12 +28,13 @@ describe LogStash::Outputs::Redis do
27
28
  redis.register
28
29
  expect(redis).to receive(:buffer_receive).exactly(10000).times.and_call_original
29
30
  expect(redis).to receive(:flush).exactly(200).times
31
+ expect(redis).not_to receive(:on_flush_error)
30
32
 
31
33
  # I was able to reproduce the LocalJumpError: unexpected next exception at around 50
32
34
  # consicutive invocations. setting to 10000 should reproduce it for any environment
33
35
  # I have no clue at this point why this problem does not happen at every invocation
34
- 1.upto(10000) do
35
- expect{redis.receive(event)}.to_not raise_error
36
+ 10000.times do |i|
37
+ expect{redis.receive(LogStash::Event.new({"message" => "test-#{i}"}))}.to_not raise_error
36
38
  end
37
39
  end
38
40
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-redis
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-13 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ files:
126
126
  - LICENSE
127
127
  - NOTICE.TXT
128
128
  - README.md
129
+ - docs/index.asciidoc
129
130
  - lib/logstash/outputs/redis.rb
130
131
  - logstash-output-redis.gemspec
131
132
  - spec/integration/outputs/redis_spec.rb