logstash-input-http_poller 5.1.0 → 5.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile +2 -0
- data/docs/index.asciidoc +35 -0
- data/lib/logstash/inputs/http_poller.rb +7 -5
- data/logstash-input-http_poller.gemspec +4 -5
- data/spec/inputs/http_poller_spec.rb +64 -65
- metadata +8 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59101199a9b6eda278187132d5e5fd719247a893a29917fb305178347f89bf1e
|
4
|
+
data.tar.gz: 10f4717d83cd6befd62a314939c27db5633e65cd7c83706478efe3c2f42b1983
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e6eb58b9354d0e60aa5c2d5c1217f8fa769c3c70b8f207c453645b8e12dadd5b533193c8fcb60689319be0d6abdcba5e2de931f956a6b7974739d1cc516dad35
|
7
|
+
data.tar.gz: 15dcd344b59b3a14184153b82f5be9375427a5ff3cbdf2103922a4c8800624957ed86bff73c45bdf854ac52577e7a899f35c750d123ca8b8fc870e005e36cd5a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## 5.3.0
|
2
|
+
- Feat: added ssl_supported_protocols option [#133](https://github.com/logstash-plugins/logstash-input-http_poller/pull/133)
|
3
|
+
|
4
|
+
## 5.2.1
|
5
|
+
- Deps: unpin rufus-scheduler dependency [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
|
6
|
+
|
7
|
+
## 5.2.0
|
8
|
+
- Feat: support ssl_verification_mode option [#131](https://github.com/logstash-plugins/logstash-input-http_poller/pull/131)
|
9
|
+
|
1
10
|
## 5.1.0
|
2
11
|
- Add ECS support [#129](https://github.com/logstash-plugins/logstash-input-http_poller/pull/129)
|
3
12
|
|
data/Gemfile
CHANGED
@@ -9,3 +9,5 @@ if Dir.exist?(logstash_path) && use_logstash_source
|
|
9
9
|
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
10
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
11
|
end
|
12
|
+
|
13
|
+
gem 'rufus-scheduler', ENV['RUFUS_SCHEDULER_VERSION'] if ENV['RUFUS_SCHEDULER_VERSION']
|
data/docs/index.asciidoc
CHANGED
@@ -146,6 +146,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
146
146
|
| <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
|
147
147
|
| <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
|
148
148
|
| <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
|
149
|
+
| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
|
150
|
+
| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
|
149
151
|
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
150
152
|
| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
|
151
153
|
| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
|
@@ -413,6 +415,39 @@ See: rufus/scheduler for details about different schedule options and value stri
|
|
413
415
|
|
414
416
|
Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
415
417
|
|
418
|
+
[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
|
419
|
+
===== `ssl_supported_protocols`
|
420
|
+
|
421
|
+
* Value type is <<string,string>>
|
422
|
+
* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
|
423
|
+
* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
|
424
|
+
`'TLSv1.1'` is not considered secure and is only provided for legacy applications.
|
425
|
+
|
426
|
+
List of allowed SSL/TLS versions to use when establishing a connection to the HTTP endpoint.
|
427
|
+
|
428
|
+
For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
|
429
|
+
`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
|
430
|
+
|
431
|
+
NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
|
432
|
+
the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
|
433
|
+
the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
|
434
|
+
|
435
|
+
[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
|
436
|
+
===== `ssl_verification_mode`
|
437
|
+
|
438
|
+
* Value type is <<string,string>>
|
439
|
+
* Supported values are: `full`, `none`
|
440
|
+
* Default value is `full`
|
441
|
+
|
442
|
+
Controls the verification of server certificates.
|
443
|
+
The `full` option verifies that the provided certificate is signed by a trusted authority (CA)
|
444
|
+
and also that the server’s hostname (or IP address) matches the names identified within the certificate.
|
445
|
+
|
446
|
+
The `none` setting performs no verification of the server’s certificate.
|
447
|
+
This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration.
|
448
|
+
It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors.
|
449
|
+
Using `none` in production environments is strongly discouraged.
|
450
|
+
|
416
451
|
[id="plugins-{type}s-{plugin}-target"]
|
417
452
|
===== `target`
|
418
453
|
|
@@ -55,9 +55,12 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
55
55
|
setup_requests!
|
56
56
|
end
|
57
57
|
|
58
|
+
# @overload
|
58
59
|
def stop
|
59
|
-
|
60
|
-
|
60
|
+
if @scheduler
|
61
|
+
@scheduler.shutdown # on newer Rufus (3.8) this joins on the scheduler thread
|
62
|
+
end
|
63
|
+
# TODO implement client.close as we as releasing it's pooled resources!
|
61
64
|
end
|
62
65
|
|
63
66
|
private
|
@@ -163,16 +166,15 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
163
166
|
#schedule hash must contain exactly one of the allowed keys
|
164
167
|
msg_invalid_schedule = "Invalid config. schedule hash must contain " +
|
165
168
|
"exactly one of the following keys - cron, at, every or in"
|
166
|
-
raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
|
169
|
+
raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
|
167
170
|
schedule_type = @schedule.keys.first
|
168
171
|
schedule_value = @schedule[schedule_type]
|
169
172
|
raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
|
170
173
|
|
171
174
|
@scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
|
172
|
-
#as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
|
173
175
|
opts = schedule_type == "every" ? { :first_in => 0.01 } : {}
|
174
176
|
@scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
|
175
|
-
@scheduler.join
|
177
|
+
@scheduler.thread.join # due newer rufus (3.8) doing a blocking operation on scheduler.join
|
176
178
|
end
|
177
179
|
|
178
180
|
def run_once(queue)
|
@@ -1,9 +1,9 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-http_poller'
|
3
|
-
s.version
|
3
|
+
s.version = '5.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Decodes the output of an HTTP API into events"
|
6
|
-
s.description
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = [ "Elastic", "andrewvc"]
|
8
8
|
s.email = 'info@elastic.co'
|
9
9
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
@@ -20,9 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
|
-
s.add_runtime_dependency
|
24
|
-
s.add_runtime_dependency '
|
25
|
-
s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
|
23
|
+
s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0"
|
24
|
+
s.add_runtime_dependency 'rufus-scheduler', ">= 3.0.9"
|
26
25
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
|
27
26
|
s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
|
28
27
|
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|
@@ -25,18 +25,19 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
25
25
|
"schedule" => default_schedule,
|
26
26
|
"urls" => default_urls,
|
27
27
|
"codec" => "json",
|
28
|
-
"metadata_target" => metadata_target
|
28
|
+
"metadata_target" => metadata_target,
|
29
|
+
"pool_max" => 3, "pool_max_per_route" => 1, 'keepalive' => false
|
29
30
|
}
|
30
31
|
}
|
31
|
-
let(:
|
32
|
+
let(:opts) { default_opts }
|
32
33
|
|
34
|
+
subject(:plugin) { described_class.new(opts) }
|
33
35
|
|
34
36
|
describe "instances" do
|
35
|
-
subject {
|
37
|
+
subject { described_class.new(default_opts) }
|
36
38
|
|
37
|
-
before
|
38
|
-
|
39
|
-
end
|
39
|
+
before { subject.register }
|
40
|
+
after { subject.stop }
|
40
41
|
|
41
42
|
describe "#run" do
|
42
43
|
it "should setup a scheduler" do
|
@@ -189,22 +190,21 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
189
190
|
"metadata_target" => metadata_target
|
190
191
|
}
|
191
192
|
}
|
192
|
-
|
193
|
-
|
194
|
-
instance.register
|
193
|
+
|
194
|
+
before do
|
195
195
|
Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
|
196
196
|
Timecop.scale(60)
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
end
|
201
|
-
sleep 3
|
202
|
-
instance.stop
|
203
|
-
runner.kill
|
204
|
-
runner.join
|
205
|
-
expect(queue.size).to eq(2)
|
197
|
+
end
|
198
|
+
|
199
|
+
after do
|
206
200
|
Timecop.return
|
207
201
|
end
|
202
|
+
|
203
|
+
it "should run at the schedule" do
|
204
|
+
run_plugin_and_yield_queue(plugin, sleep: 3) do |queue|
|
205
|
+
try(5) { expect(queue.size).to be >= 2 }
|
206
|
+
end
|
207
|
+
end
|
208
208
|
end
|
209
209
|
|
210
210
|
context "given 'at' expression" do
|
@@ -216,22 +216,21 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
216
216
|
"metadata_target" => metadata_target
|
217
217
|
}
|
218
218
|
}
|
219
|
-
|
220
|
-
|
221
|
-
instance.register
|
219
|
+
|
220
|
+
before do
|
222
221
|
Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
|
223
|
-
Timecop.scale(60 * 5)
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
end
|
228
|
-
sleep 2
|
229
|
-
instance.stop
|
230
|
-
runner.kill
|
231
|
-
runner.join
|
232
|
-
expect(queue.size).to eq(1)
|
222
|
+
Timecop.scale (60 * 5) / 2
|
223
|
+
end
|
224
|
+
|
225
|
+
after do
|
233
226
|
Timecop.return
|
234
227
|
end
|
228
|
+
|
229
|
+
it "should run at the schedule" do
|
230
|
+
run_plugin_and_yield_queue(plugin, sleep: 2) do |queue|
|
231
|
+
try(5) { expect(queue.size).to eq(1) }
|
232
|
+
end
|
233
|
+
end
|
235
234
|
end
|
236
235
|
|
237
236
|
context "given 'every' expression" do
|
@@ -244,20 +243,12 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
244
243
|
}
|
245
244
|
}
|
246
245
|
it "should run at the schedule" do
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
246
|
+
run_plugin_and_yield_queue(plugin, sleep: 5) do |queue|
|
247
|
+
#T 0123456
|
248
|
+
#events x x x x
|
249
|
+
#expects 3 events at T=5
|
250
|
+
try(5) { expect(queue.size).to be_between(2, 3) }
|
252
251
|
end
|
253
|
-
#T 0123456
|
254
|
-
#events x x x x
|
255
|
-
#expects 3 events at T=5
|
256
|
-
sleep 5
|
257
|
-
instance.stop
|
258
|
-
runner.kill
|
259
|
-
runner.join
|
260
|
-
expect(queue.size).to be_between(2, 3)
|
261
252
|
end
|
262
253
|
end
|
263
254
|
|
@@ -271,21 +262,28 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
271
262
|
}
|
272
263
|
}
|
273
264
|
it "should run at the schedule" do
|
274
|
-
|
275
|
-
|
276
|
-
queue = Queue.new
|
277
|
-
runner = Thread.new do
|
278
|
-
instance.run(queue)
|
265
|
+
run_plugin_and_yield_queue(plugin, sleep: 2.5) do |queue|
|
266
|
+
try(5) { expect(queue.size).to eq(1) }
|
279
267
|
end
|
280
|
-
sleep 3
|
281
|
-
instance.stop
|
282
|
-
runner.kill
|
283
|
-
runner.join
|
284
|
-
expect(queue.size).to eq(1)
|
285
268
|
end
|
286
269
|
end
|
287
270
|
end
|
288
271
|
|
272
|
+
def run_plugin_and_yield_queue(plugin, sleep: nil)
|
273
|
+
plugin.register
|
274
|
+
queue = Queue.new
|
275
|
+
begin
|
276
|
+
runner = Thread.new do
|
277
|
+
plugin.run(queue)
|
278
|
+
end
|
279
|
+
sleep(sleep) if sleep
|
280
|
+
yield(queue)
|
281
|
+
ensure
|
282
|
+
plugin.stop
|
283
|
+
runner.join if runner
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
289
287
|
describe "events", :ecs_compatibility_support, :aggregate_failures do
|
290
288
|
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
291
289
|
before do
|
@@ -339,6 +337,8 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
339
337
|
event # materialize the subject
|
340
338
|
end
|
341
339
|
|
340
|
+
after { poller.stop }
|
341
|
+
|
342
342
|
it "should enqueue a message" do
|
343
343
|
expect(event).to be_a(LogStash::Event)
|
344
344
|
end
|
@@ -399,9 +399,6 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
399
399
|
let(:payload) { {"a" => 2, "hello" => ["a", "b", "c"]} }
|
400
400
|
let(:response_body) { LogStash::Json.dump(payload) }
|
401
401
|
let(:opts) { default_opts }
|
402
|
-
let(:instance) {
|
403
|
-
klass.new(opts)
|
404
|
-
}
|
405
402
|
let(:name) { default_name }
|
406
403
|
let(:url) { default_url }
|
407
404
|
let(:code) { 202 }
|
@@ -411,14 +408,14 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
411
408
|
}
|
412
409
|
|
413
410
|
before do
|
414
|
-
|
411
|
+
plugin.register
|
415
412
|
u = url.is_a?(Hash) ? url["url"] : url # handle both complex specs and simple string URLs
|
416
|
-
|
413
|
+
plugin.client.stub(u,
|
417
414
|
:body => response_body,
|
418
415
|
:code => code
|
419
416
|
)
|
420
|
-
allow(
|
421
|
-
|
417
|
+
allow(plugin).to receive(:decorate)
|
418
|
+
plugin.send(:run_once, queue)
|
422
419
|
end
|
423
420
|
|
424
421
|
it "should have a matching message" do
|
@@ -426,7 +423,7 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
426
423
|
end
|
427
424
|
|
428
425
|
it "should decorate the event" do
|
429
|
-
expect(
|
426
|
+
expect(plugin).to have_received(:decorate).once
|
430
427
|
end
|
431
428
|
|
432
429
|
include_examples("matching metadata")
|
@@ -434,7 +431,7 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
434
431
|
context "with an empty body" do
|
435
432
|
let(:response_body) { "" }
|
436
433
|
it "should return an empty event" do
|
437
|
-
|
434
|
+
plugin.send(:run_once, queue)
|
438
435
|
headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]",
|
439
436
|
v1: "[#{metadata_target}][input][http_poller][response][headers]"]
|
440
437
|
expect(event.get("#{headers_field}[content-length]")).to eql("0")
|
@@ -449,7 +446,7 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
449
446
|
}
|
450
447
|
|
451
448
|
it "should not have any metadata on the event" do
|
452
|
-
|
449
|
+
plugin.send(:run_once, queue)
|
453
450
|
expect(event.get(metadata_target)).to be_nil
|
454
451
|
end
|
455
452
|
end
|
@@ -555,6 +552,8 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
555
552
|
|
556
553
|
describe "stopping" do
|
557
554
|
let(:config) { default_opts }
|
558
|
-
it_behaves_like "an interruptible input plugin"
|
555
|
+
it_behaves_like "an interruptible input plugin" do
|
556
|
+
let(:allowed_lag) { 10 } # CI: wait till scheduler shuts down
|
557
|
+
end
|
559
558
|
end
|
560
559
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-http_poller
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-03-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -48,35 +48,21 @@ dependencies:
|
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
|
-
- - "
|
51
|
+
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
53
|
+
version: 7.2.0
|
54
54
|
name: logstash-mixin-http_client
|
55
55
|
prerelease: false
|
56
56
|
type: :runtime
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '7'
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
requirement: !ruby/object:Gem::Requirement
|
64
|
-
requirements:
|
65
|
-
- - "~>"
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
version: 0.0.22
|
68
|
-
name: stud
|
69
|
-
prerelease: false
|
70
|
-
type: :runtime
|
71
|
-
version_requirements: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
59
|
+
- - ">="
|
74
60
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
61
|
+
version: 7.2.0
|
76
62
|
- !ruby/object:Gem::Dependency
|
77
63
|
requirement: !ruby/object:Gem::Requirement
|
78
64
|
requirements:
|
79
|
-
- - "
|
65
|
+
- - ">="
|
80
66
|
- !ruby/object:Gem::Version
|
81
67
|
version: 3.0.9
|
82
68
|
name: rufus-scheduler
|
@@ -84,7 +70,7 @@ dependencies:
|
|
84
70
|
type: :runtime
|
85
71
|
version_requirements: !ruby/object:Gem::Requirement
|
86
72
|
requirements:
|
87
|
-
- - "
|
73
|
+
- - ">="
|
88
74
|
- !ruby/object:Gem::Version
|
89
75
|
version: 3.0.9
|
90
76
|
- !ruby/object:Gem::Dependency
|