logstash-input-beats 2.0.3 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/lib/logstash/inputs/beats.rb +115 -101
- data/lib/logstash/{circuit_breaker.rb → inputs/beats_support/circuit_breaker.rb} +16 -10
- data/lib/logstash/inputs/beats_support/codec_callback_listener.rb +26 -0
- data/lib/logstash/inputs/beats_support/connection_handler.rb +79 -0
- data/lib/logstash/inputs/beats_support/decoded_event_transform.rb +34 -0
- data/lib/logstash/inputs/beats_support/event_transform_common.rb +40 -0
- data/lib/logstash/inputs/beats_support/raw_event_transform.rb +18 -0
- data/lib/logstash/inputs/beats_support/synchronous_queue_with_offer.rb +36 -0
- data/lib/lumberjack/beats/server.rb +58 -11
- data/logstash-input-beats.gemspec +4 -3
- data/spec/inputs/beats_spec.rb +35 -126
- data/spec/{logstash → inputs/beats_support}/circuit_breaker_spec.rb +11 -10
- data/spec/inputs/beats_support/codec_callback_listener_spec.rb +52 -0
- data/spec/inputs/beats_support/connection_handler_spec.rb +93 -0
- data/spec/inputs/beats_support/decoded_event_transform_spec.rb +67 -0
- data/spec/inputs/beats_support/event_transform_common_spec.rb +11 -0
- data/spec/inputs/beats_support/raw_event_transform_spec.rb +26 -0
- data/spec/integration_spec.rb +22 -12
- data/spec/lumberjack/beats/server_spec.rb +3 -3
- data/spec/support/logstash_test.rb +25 -0
- data/spec/support/shared_examples.rb +56 -0
- metadata +74 -45
- data/lib/logstash/sized_queue_timeout.rb +0 -64
- data/spec/logstash/size_queue_timeout_spec.rb +0 -100
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-beats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,146 +28,162 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-plain
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-codec-plain
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: concurrent-ruby
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
|
-
- -
|
51
|
+
- - ~>
|
45
52
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
47
|
-
- !ruby/object:Gem::Dependency
|
53
|
+
version: 0.9.2
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - ~>
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: 0.9.2
|
53
|
-
name: concurrent-ruby
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: thread_safe
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
65
|
- - ~>
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0.
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: 0.3.5
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - ~>
|
65
71
|
- !ruby/object:Gem::Version
|
66
|
-
version:
|
67
|
-
name: logstash-codec-multiline
|
72
|
+
version: 0.3.5
|
68
73
|
prerelease: false
|
69
74
|
type: :runtime
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: logstash-codec-multiline
|
70
77
|
version_requirements: !ruby/object:Gem::Requirement
|
71
78
|
requirements:
|
72
79
|
- - ~>
|
73
80
|
- !ruby/object:Gem::Version
|
74
|
-
version: 2.0.
|
75
|
-
- !ruby/object:Gem::Dependency
|
81
|
+
version: 2.0.5
|
76
82
|
requirement: !ruby/object:Gem::Requirement
|
77
83
|
requirements:
|
78
84
|
- - ~>
|
79
85
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
81
|
-
name: flores
|
86
|
+
version: 2.0.5
|
82
87
|
prerelease: false
|
83
|
-
type: :
|
88
|
+
type: :runtime
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: flores
|
84
91
|
version_requirements: !ruby/object:Gem::Requirement
|
85
92
|
requirements:
|
86
93
|
- - ~>
|
87
94
|
- !ruby/object:Gem::Version
|
88
95
|
version: 0.0.6
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
96
|
requirement: !ruby/object:Gem::Requirement
|
91
97
|
requirements:
|
92
|
-
- -
|
98
|
+
- - ~>
|
93
99
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
95
|
-
name: rspec
|
100
|
+
version: 0.0.6
|
96
101
|
prerelease: false
|
97
102
|
type: :development
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: rspec
|
98
105
|
version_requirements: !ruby/object:Gem::Requirement
|
99
106
|
requirements:
|
100
107
|
- - '>='
|
101
108
|
- !ruby/object:Gem::Version
|
102
109
|
version: '0'
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
110
|
requirement: !ruby/object:Gem::Requirement
|
105
111
|
requirements:
|
106
112
|
- - '>='
|
107
113
|
- !ruby/object:Gem::Version
|
108
114
|
version: '0'
|
109
|
-
name: stud
|
110
115
|
prerelease: false
|
111
116
|
type: :development
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: stud
|
112
119
|
version_requirements: !ruby/object:Gem::Requirement
|
113
120
|
requirements:
|
114
121
|
- - '>='
|
115
122
|
- !ruby/object:Gem::Version
|
116
123
|
version: '0'
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
124
|
requirement: !ruby/object:Gem::Requirement
|
119
125
|
requirements:
|
120
126
|
- - '>='
|
121
127
|
- !ruby/object:Gem::Version
|
122
128
|
version: '0'
|
123
|
-
name: pry
|
124
129
|
prerelease: false
|
125
130
|
type: :development
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: pry
|
126
133
|
version_requirements: !ruby/object:Gem::Requirement
|
127
134
|
requirements:
|
128
135
|
- - '>='
|
129
136
|
- !ruby/object:Gem::Version
|
130
137
|
version: '0'
|
131
|
-
- !ruby/object:Gem::Dependency
|
132
138
|
requirement: !ruby/object:Gem::Requirement
|
133
139
|
requirements:
|
134
140
|
- - '>='
|
135
141
|
- !ruby/object:Gem::Version
|
136
142
|
version: '0'
|
137
|
-
name: rspec-wait
|
138
143
|
prerelease: false
|
139
144
|
type: :development
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: rspec-wait
|
140
147
|
version_requirements: !ruby/object:Gem::Requirement
|
141
148
|
requirements:
|
142
149
|
- - '>='
|
143
150
|
- !ruby/object:Gem::Version
|
144
151
|
version: '0'
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
152
|
requirement: !ruby/object:Gem::Requirement
|
147
153
|
requirements:
|
148
|
-
- -
|
154
|
+
- - '>='
|
149
155
|
- !ruby/object:Gem::Version
|
150
|
-
version: 0
|
151
|
-
name: logstash-devutils
|
156
|
+
version: '0'
|
152
157
|
prerelease: false
|
153
158
|
type: :development
|
159
|
+
- !ruby/object:Gem::Dependency
|
160
|
+
name: logstash-devutils
|
154
161
|
version_requirements: !ruby/object:Gem::Requirement
|
155
162
|
requirements:
|
156
163
|
- - ~>
|
157
164
|
- !ruby/object:Gem::Version
|
158
165
|
version: 0.0.18
|
159
|
-
- !ruby/object:Gem::Dependency
|
160
166
|
requirement: !ruby/object:Gem::Requirement
|
161
167
|
requirements:
|
162
|
-
- -
|
168
|
+
- - ~>
|
163
169
|
- !ruby/object:Gem::Version
|
164
|
-
version:
|
165
|
-
name: logstash-codec-json
|
170
|
+
version: 0.0.18
|
166
171
|
prerelease: false
|
167
172
|
type: :development
|
173
|
+
- !ruby/object:Gem::Dependency
|
174
|
+
name: logstash-codec-json
|
168
175
|
version_requirements: !ruby/object:Gem::Requirement
|
169
176
|
requirements:
|
170
177
|
- - '>='
|
171
178
|
- !ruby/object:Gem::Version
|
172
179
|
version: '0'
|
180
|
+
requirement: !ruby/object:Gem::Requirement
|
181
|
+
requirements:
|
182
|
+
- - '>='
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
version: '0'
|
185
|
+
prerelease: false
|
186
|
+
type: :development
|
173
187
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
174
188
|
email: info@elastic.co
|
175
189
|
executables: []
|
@@ -183,17 +197,26 @@ files:
|
|
183
197
|
- NOTICE.TXT
|
184
198
|
- PROTOCOL.md
|
185
199
|
- README.md
|
186
|
-
- lib/logstash/circuit_breaker.rb
|
187
200
|
- lib/logstash/inputs/beats.rb
|
188
|
-
- lib/logstash/
|
201
|
+
- lib/logstash/inputs/beats_support/circuit_breaker.rb
|
202
|
+
- lib/logstash/inputs/beats_support/codec_callback_listener.rb
|
203
|
+
- lib/logstash/inputs/beats_support/connection_handler.rb
|
204
|
+
- lib/logstash/inputs/beats_support/decoded_event_transform.rb
|
205
|
+
- lib/logstash/inputs/beats_support/event_transform_common.rb
|
206
|
+
- lib/logstash/inputs/beats_support/raw_event_transform.rb
|
207
|
+
- lib/logstash/inputs/beats_support/synchronous_queue_with_offer.rb
|
189
208
|
- lib/lumberjack/beats.rb
|
190
209
|
- lib/lumberjack/beats/client.rb
|
191
210
|
- lib/lumberjack/beats/server.rb
|
192
211
|
- logstash-input-beats.gemspec
|
193
212
|
- spec/inputs/beats_spec.rb
|
213
|
+
- spec/inputs/beats_support/circuit_breaker_spec.rb
|
214
|
+
- spec/inputs/beats_support/codec_callback_listener_spec.rb
|
215
|
+
- spec/inputs/beats_support/connection_handler_spec.rb
|
216
|
+
- spec/inputs/beats_support/decoded_event_transform_spec.rb
|
217
|
+
- spec/inputs/beats_support/event_transform_common_spec.rb
|
218
|
+
- spec/inputs/beats_support/raw_event_transform_spec.rb
|
194
219
|
- spec/integration_spec.rb
|
195
|
-
- spec/logstash/circuit_breaker_spec.rb
|
196
|
-
- spec/logstash/size_queue_timeout_spec.rb
|
197
220
|
- spec/lumberjack/beats/acking_protocol_v1_spec.rb
|
198
221
|
- spec/lumberjack/beats/acking_protocol_v2_spec.rb
|
199
222
|
- spec/lumberjack/beats/client_spec.rb
|
@@ -201,6 +224,7 @@ files:
|
|
201
224
|
- spec/lumberjack/beats/server_spec.rb
|
202
225
|
- spec/spec_helper.rb
|
203
226
|
- spec/support/logstash_test.rb
|
227
|
+
- spec/support/shared_examples.rb
|
204
228
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
205
229
|
licenses:
|
206
230
|
- Apache License (2.0)
|
@@ -229,9 +253,13 @@ specification_version: 4
|
|
229
253
|
summary: Receive events using the lumberjack protocol.
|
230
254
|
test_files:
|
231
255
|
- spec/inputs/beats_spec.rb
|
256
|
+
- spec/inputs/beats_support/circuit_breaker_spec.rb
|
257
|
+
- spec/inputs/beats_support/codec_callback_listener_spec.rb
|
258
|
+
- spec/inputs/beats_support/connection_handler_spec.rb
|
259
|
+
- spec/inputs/beats_support/decoded_event_transform_spec.rb
|
260
|
+
- spec/inputs/beats_support/event_transform_common_spec.rb
|
261
|
+
- spec/inputs/beats_support/raw_event_transform_spec.rb
|
232
262
|
- spec/integration_spec.rb
|
233
|
-
- spec/logstash/circuit_breaker_spec.rb
|
234
|
-
- spec/logstash/size_queue_timeout_spec.rb
|
235
263
|
- spec/lumberjack/beats/acking_protocol_v1_spec.rb
|
236
264
|
- spec/lumberjack/beats/acking_protocol_v2_spec.rb
|
237
265
|
- spec/lumberjack/beats/client_spec.rb
|
@@ -239,3 +267,4 @@ test_files:
|
|
239
267
|
- spec/lumberjack/beats/server_spec.rb
|
240
268
|
- spec/spec_helper.rb
|
241
269
|
- spec/support/logstash_test.rb
|
270
|
+
- spec/support/shared_examples.rb
|
@@ -1,64 +0,0 @@
|
|
1
|
-
require "thread"
|
2
|
-
require "concurrent"
|
3
|
-
|
4
|
-
module LogStash
|
5
|
-
# Minimal subset implement of a SizedQueue supporting
|
6
|
-
# a timeout option on the lock.
|
7
|
-
#
|
8
|
-
# This will be part of the main Logstash's sized queue
|
9
|
-
class SizedQueueTimeout
|
10
|
-
class TimeoutError < StandardError; end
|
11
|
-
|
12
|
-
DEFAULT_TIMEOUT = 5 # in seconds
|
13
|
-
|
14
|
-
def initialize(max_size, options = {})
|
15
|
-
# `concurrent-ruby` are deprecating the `Condition`
|
16
|
-
# in favor of a Synchonization class that you need to implement.
|
17
|
-
# this was bit overkill to only check if the wait did a timeout.
|
18
|
-
@condition_in = ConditionVariable.new
|
19
|
-
@condition_out = ConditionVariable.new
|
20
|
-
|
21
|
-
@max_size = max_size
|
22
|
-
@queue = []
|
23
|
-
@mutex = Mutex.new
|
24
|
-
end
|
25
|
-
|
26
|
-
def push(obj, timeout = DEFAULT_TIMEOUT)
|
27
|
-
@mutex.synchronize do
|
28
|
-
while full? # wake up check
|
29
|
-
start_time = Concurrent.monotonic_time
|
30
|
-
@condition_out.wait(@mutex, timeout)
|
31
|
-
if start_time + timeout - Concurrent.monotonic_time < 0
|
32
|
-
raise TimeoutError
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
@queue << obj
|
37
|
-
@condition_in.signal
|
38
|
-
|
39
|
-
return obj
|
40
|
-
end
|
41
|
-
end
|
42
|
-
alias_method :<<, :push
|
43
|
-
|
44
|
-
def size
|
45
|
-
@mutex.synchronize { @queue.size }
|
46
|
-
end
|
47
|
-
|
48
|
-
def pop_no_timeout
|
49
|
-
@mutex.synchronize do
|
50
|
-
@condition_in.wait(@mutex) while @queue.empty? # Wake up check
|
51
|
-
|
52
|
-
obj = @queue.shift
|
53
|
-
@condition_out.signal
|
54
|
-
|
55
|
-
return obj
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
private
|
60
|
-
def full?
|
61
|
-
@queue.size == @max_size
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
@@ -1,100 +0,0 @@
|
|
1
|
-
require_relative "../spec_helper"
|
2
|
-
require "logstash/sized_queue_timeout"
|
3
|
-
require "flores/random"
|
4
|
-
require "stud/try"
|
5
|
-
|
6
|
-
describe "LogStash::SizedQueueTimeout" do
|
7
|
-
let(:max_size) { Flores::Random.integer(2..100) }
|
8
|
-
let(:element) { Flores::Random.text(0..100) }
|
9
|
-
|
10
|
-
subject { LogStash::SizedQueueTimeout.new(max_size) }
|
11
|
-
|
12
|
-
it "adds element to the queue" do
|
13
|
-
subject << element
|
14
|
-
expect(subject.size).to eq(1)
|
15
|
-
end
|
16
|
-
|
17
|
-
it "allow to pop element from the queue" do
|
18
|
-
subject << element
|
19
|
-
subject << "awesome"
|
20
|
-
|
21
|
-
expect(subject.pop_no_timeout).to eq(element)
|
22
|
-
end
|
23
|
-
|
24
|
-
context "when the queue is full" do
|
25
|
-
before do
|
26
|
-
max_size.times { subject << element }
|
27
|
-
end
|
28
|
-
|
29
|
-
it "block with a timeout" do
|
30
|
-
expect {
|
31
|
-
subject << element
|
32
|
-
}.to raise_error(LogStash::SizedQueueTimeout::TimeoutError)
|
33
|
-
end
|
34
|
-
|
35
|
-
it "unblock when we pop" do
|
36
|
-
blocked = Thread.new do
|
37
|
-
subject << element
|
38
|
-
end
|
39
|
-
sleep(0.1) until blocked.stop?
|
40
|
-
|
41
|
-
expect(blocked.status).to eq("sleep")
|
42
|
-
|
43
|
-
th = Thread.new do
|
44
|
-
subject.pop_no_timeout
|
45
|
-
end
|
46
|
-
sleep(0.1) until th.stop?
|
47
|
-
|
48
|
-
expect(blocked.status).to eq(false)
|
49
|
-
|
50
|
-
blocked.join
|
51
|
-
th.join
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
context "when the queue is empty" do
|
56
|
-
it "block on pop" do
|
57
|
-
blocked = Thread.new do
|
58
|
-
subject.pop_no_timeout
|
59
|
-
end
|
60
|
-
sleep(0.1) until blocked.stop?
|
61
|
-
|
62
|
-
expect(blocked.status).to eq("sleep")
|
63
|
-
|
64
|
-
th = Thread.new do
|
65
|
-
subject << element
|
66
|
-
end
|
67
|
-
sleep(0.1) until th.stop?
|
68
|
-
|
69
|
-
expect(blocked.status).to eq(false)
|
70
|
-
th.join
|
71
|
-
blocked.join
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
context "when the queue is occupied but not full" do
|
76
|
-
before :each do
|
77
|
-
Flores::Random.iterations(1..max_size-1) { subject << "hurray" }
|
78
|
-
end
|
79
|
-
|
80
|
-
it "doesnt block on pop" do
|
81
|
-
th = Thread.new do
|
82
|
-
subject.pop_no_timeout
|
83
|
-
end
|
84
|
-
sleep(0.1) until th.stop?
|
85
|
-
|
86
|
-
expect(th.status).to eq(false)
|
87
|
-
th.join
|
88
|
-
end
|
89
|
-
|
90
|
-
it "doesnt block on push" do
|
91
|
-
th = Thread.new do
|
92
|
-
subject << element
|
93
|
-
end
|
94
|
-
sleep(0.1) until th.stop?
|
95
|
-
|
96
|
-
expect(th.status).to eq(false)
|
97
|
-
th.join
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|