logstash-input-http_poller 5.0.1 → 5.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -1
- data/Gemfile +2 -0
- data/LICENSE +3 -3
- data/README.md +1 -1
- data/docs/index.asciidoc +176 -20
- data/lib/logstash/inputs/http_poller.rb +87 -42
- data/logstash-input-http_poller.gemspec +6 -4
- data/spec/inputs/http_poller_spec.rb +277 -224
- metadata +46 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffa63b5111349fab5d81dbc52cc005ab2ab6ed296aab3135859d941b3fd32c4b
|
4
|
+
data.tar.gz: 6ab7912818139b2c9595fe7b0797f0c4e02bb9af2722a32e5af1c9ca8c887536
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2093f0da0658436437293283a8425e2e497d2bb1e17a9b3910dff3e2ea5bf8a1ce546225abf4f97f24866abd812d0078e0c479ee7c3ef401ee3c3ec611002c21
|
7
|
+
data.tar.gz: 24210d613caee8ed465816b9865d75ad0547d8e81f5160858742d77bbe4ddef46284b798b4c1581bc706f0267f4316718318995dc5dbb3647c123d407d7fdf08
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
## 5.2.1
|
2
|
+
- Deps: unpin rufus-scheduler dependency [#130](https://github.com/logstash-plugins/logstash-input-http_poller/pull/130)
|
3
|
+
|
4
|
+
## 5.2.0
|
5
|
+
- Feat: support ssl_verification_mode option [#131](https://github.com/logstash-plugins/logstash-input-http_poller/pull/131)
|
6
|
+
|
7
|
+
## 5.1.0
|
8
|
+
- Add ECS support [#129](https://github.com/logstash-plugins/logstash-input-http_poller/pull/129)
|
9
|
+
|
10
|
+
## 5.0.2
|
11
|
+
- [DOC]Expanded url option to include Manticore keys [#119](https://github.com/logstash-plugins/logstash-input-http_poller/pull/119)
|
12
|
+
|
1
13
|
## 5.0.1
|
2
14
|
- Fixed minor doc and doc formatting issues [#107](https://github.com/logstash-plugins/logstash-input-http_poller/pull/107)
|
3
15
|
|
@@ -18,7 +30,7 @@
|
|
18
30
|
- Docs: Remove row in overview table to fix build error
|
19
31
|
|
20
32
|
## 4.0.2
|
21
|
-
- Don't bleed URLs credentials on startup and on exception #82
|
33
|
+
- Don't bleed URLs credentials on startup and on exception [#82](https://github.com/logstash-plugins/logstash-input-http_poller/pull/82)
|
22
34
|
|
23
35
|
## 4.0.1
|
24
36
|
- Fix some documentation issues
|
data/Gemfile
CHANGED
@@ -9,3 +9,5 @@ if Dir.exist?(logstash_path) && use_logstash_source
|
|
9
9
|
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
10
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
11
|
end
|
12
|
+
|
13
|
+
gem 'rufus-scheduler', ENV['RUFUS_SCHEDULER_VERSION'] if ENV['RUFUS_SCHEDULER_VERSION']
|
data/LICENSE
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
|
1
2
|
Apache License
|
2
3
|
Version 2.0, January 2004
|
3
4
|
http://www.apache.org/licenses/
|
@@ -178,7 +179,7 @@
|
|
178
179
|
APPENDIX: How to apply the Apache License to your work.
|
179
180
|
|
180
181
|
To apply the Apache License to your work, attach the following
|
181
|
-
boilerplate notice, with the fields enclosed by brackets "
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
182
183
|
replaced with your own identifying information. (Don't include
|
183
184
|
the brackets!) The text should be enclosed in the appropriate
|
184
185
|
comment syntax for the file format. We also recommend that a
|
@@ -186,7 +187,7 @@
|
|
186
187
|
same "printed page" as the copyright notice for easier
|
187
188
|
identification within third-party archives.
|
188
189
|
|
189
|
-
Copyright
|
190
|
+
Copyright 2020 Elastic and contributors
|
190
191
|
|
191
192
|
Licensed under the Apache License, Version 2.0 (the "License");
|
192
193
|
you may not use this file except in compliance with the License.
|
@@ -199,4 +200,3 @@
|
|
199
200
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
200
201
|
See the License for the specific language governing permissions and
|
201
202
|
limitations under the License.
|
202
|
-
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Logstash HTTP input plugin
|
2
2
|
|
3
|
-
[![Travis Build Status](https://travis-ci.
|
3
|
+
[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-http_poller.svg)](https://travis-ci.com/logstash-plugins/logstash-input-http_poller)
|
4
4
|
|
5
5
|
This plugin is based off [logstash-input-rest](https://github.com/maximede/logstash-input-rest) by @maximede.
|
6
6
|
|
data/docs/index.asciidoc
CHANGED
@@ -87,6 +87,36 @@ The above snippet will create two files `downloaded_cert.pem` and `downloaded_tr
|
|
87
87
|
----------------------------------
|
88
88
|
|
89
89
|
|
90
|
+
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
91
|
+
==== Event Metadata and the Elastic Common Schema (ECS)
|
92
|
+
|
93
|
+
This input will add metadata about the HTTP connection itself to each event.
|
94
|
+
|
95
|
+
When ECS compatibility is disabled, metadata was added to a variety of non-standard top-level fields, which has the potential to create confusion and schema conflicts downstream.
|
96
|
+
|
97
|
+
With ECS Compatibility Mode, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace.
|
98
|
+
|
99
|
+
Here’s how ECS compatibility mode affects output.
|
100
|
+
[cols="<l,<l,e,<e"]
|
101
|
+
|=======================================================================
|
102
|
+
| ECS disabled | ECS v1 | Availability | Description
|
103
|
+
|
104
|
+
| [@metadata][host] | [@metadata][input][http_poller][request][host][hostname] | Always | Hostname
|
105
|
+
| [@metadata][code] | [@metadata][input][http_poller][response][status_code] | When server responds a valid status code | HTTP response code
|
106
|
+
| [@metadata][response_headers] | [@metadata][input][http_poller][response][headers] | When server responds with headers | HTTP headers of the response
|
107
|
+
| [@metadata][response_message] | [@metadata][input][http_poller][response][status_message] | When server responds with status line | message of status line of HTTP headers
|
108
|
+
| [@metadata][runtime_seconds] | [@metadata][input][http_poller][response][elapsed_time_ns] | When server responds a valid status code | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
|
109
|
+
| [http_request_failure][runtime_seconds] | [event][duration] | When server throws exception | elapsed time of calling endpoint. ECS v1 shows in nanoseconds.
|
110
|
+
| [@metadata][times_retried] | [@metadata][input][http_poller][request][retry_count] | When the poller calls server successfully | retry count from http client library
|
111
|
+
| [@metadata][name] / [http_request_failure][name] | [@metadata][input][http_poller][request][name] | Always | The key of `urls` from poller config
|
112
|
+
| [@metadata][request] / [http_request_failure][request]| [@metadata][input][http_poller][request][original] | Always | The whole object of `urls` from poller config
|
113
|
+
| [http_request_failure][error] | [error][message] | When server throws exception | Error message
|
114
|
+
| [http_request_failure][backtrace] | [error][stack_trace] | When server throws exception | Stack trace of error
|
115
|
+
| -- | [url][full] | When server throws exception | The URL of the endpoint
|
116
|
+
| -- | [http][request][method] | When server throws exception | HTTP request method
|
117
|
+
| -- | [host][hostname] | When server throws exception | Hostname
|
118
|
+
|=======================================================================
|
119
|
+
|
90
120
|
[id="plugins-{type}s-{plugin}-options"]
|
91
121
|
==== Http_poller Input Configuration Options
|
92
122
|
|
@@ -95,20 +125,20 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
95
125
|
[cols="<,<,<",options="header",]
|
96
126
|
|=======================================================================
|
97
127
|
|Setting |Input type|Required
|
98
|
-
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|no
|
99
|
-
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
100
128
|
| <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
|
101
129
|
| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
|
102
130
|
| <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
|
103
131
|
| <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
|
104
132
|
| <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
|
105
133
|
| <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
|
134
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
106
135
|
| <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
|
107
136
|
| <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
|
108
137
|
| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
|
109
138
|
| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
|
110
139
|
| <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
|
111
140
|
| <<plugins-{type}s-{plugin}-metadata_target>> |<<string,string>>|No
|
141
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
112
142
|
| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
|
113
143
|
| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
|
114
144
|
| <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
|
@@ -116,11 +146,13 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
116
146
|
| <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
|
117
147
|
| <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
|
118
148
|
| <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
|
149
|
+
| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
|
119
150
|
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
120
151
|
| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
|
121
152
|
| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
|
122
153
|
| <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
|
123
154
|
| <<plugins-{type}s-{plugin}-urls>> |<<hash,hash>>|Yes
|
155
|
+
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
|
124
156
|
| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
|
125
157
|
|=======================================================================
|
126
158
|
|
@@ -129,23 +161,6 @@ input plugins.
|
|
129
161
|
|
130
162
|
|
131
163
|
|
132
|
-
[id="plugins-{type}s-{plugin}-user"]
|
133
|
-
===== `user`
|
134
|
-
|
135
|
-
* Value type is <<string,string>>
|
136
|
-
* There is no default value for this setting.
|
137
|
-
|
138
|
-
Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
|
139
|
-
If you set this you must also set the `password` option.
|
140
|
-
|
141
|
-
[id="plugins-{type}s-{plugin}-password"]
|
142
|
-
===== `password`
|
143
|
-
|
144
|
-
* Value type is <<password,password>>
|
145
|
-
* There is no default value for this setting.
|
146
|
-
|
147
|
-
Password to be used in conjunction with the username for HTTP authentication.
|
148
|
-
|
149
164
|
[id="plugins-{type}s-{plugin}-automatic_retries"]
|
150
165
|
===== `automatic_retries`
|
151
166
|
|
@@ -197,6 +212,81 @@ Timeout (in seconds) to wait for a connection to be established. Default is `10s
|
|
197
212
|
Enable cookie support. With this enabled the client will persist cookies
|
198
213
|
across requests as a normal web browser would. Enabled by default
|
199
214
|
|
215
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
216
|
+
===== `ecs_compatibility`
|
217
|
+
|
218
|
+
* Value type is <<string,string>>
|
219
|
+
* Supported values are:
|
220
|
+
** `disabled`: unstructured data added at root level
|
221
|
+
** `v1`: uses `error`, `url` and `http` fields that are compatible with Elastic Common Schema
|
222
|
+
|
223
|
+
Controls this plugin's compatibility with the
|
224
|
+
{ecs-ref}[Elastic Common Schema (ECS)].
|
225
|
+
See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
|
226
|
+
|
227
|
+
Example output:
|
228
|
+
|
229
|
+
**Sample output: ECS disabled**
|
230
|
+
[source,text]
|
231
|
+
-----
|
232
|
+
{
|
233
|
+
"http_poller_data" => {
|
234
|
+
"@version" => "1",
|
235
|
+
"@timestamp" => 2021-01-01T00:43:22.388Z,
|
236
|
+
"status" => "UP"
|
237
|
+
},
|
238
|
+
"@version" => "1",
|
239
|
+
"@timestamp" => 2021-01-01T00:43:22.389Z,
|
240
|
+
}
|
241
|
+
-----
|
242
|
+
|
243
|
+
**Sample output: ECS enabled**
|
244
|
+
[source,text]
|
245
|
+
-----
|
246
|
+
{
|
247
|
+
"http_poller_data" => {
|
248
|
+
"status" => "UP",
|
249
|
+
"@version" => "1",
|
250
|
+
"event" => {
|
251
|
+
"original" => "{\"status\":\"UP\"}"
|
252
|
+
},
|
253
|
+
"@timestamp" => 2021-01-01T00:40:59.558Z
|
254
|
+
},
|
255
|
+
"@version" => "1",
|
256
|
+
"@timestamp" => 2021-01-01T00:40:59.559Z
|
257
|
+
}
|
258
|
+
-----
|
259
|
+
|
260
|
+
**Sample error output: ECS enabled**
|
261
|
+
[source,text]
|
262
|
+
----
|
263
|
+
{
|
264
|
+
"@timestamp" => 2021-07-09T09:53:48.721Z,
|
265
|
+
"@version" => "1",
|
266
|
+
"host" => {
|
267
|
+
"hostname" => "MacBook-Pro"
|
268
|
+
},
|
269
|
+
"http" => {
|
270
|
+
"request" => {
|
271
|
+
"method" => "get"
|
272
|
+
}
|
273
|
+
},
|
274
|
+
"event" => {
|
275
|
+
"duration" => 259019
|
276
|
+
},
|
277
|
+
"error" => {
|
278
|
+
"stack_trace" => nil,
|
279
|
+
"message" => "Connection refused (Connection refused)"
|
280
|
+
},
|
281
|
+
"url" => {
|
282
|
+
"full" => "http://localhost:8080/actuator/health"
|
283
|
+
},
|
284
|
+
"tags" => [
|
285
|
+
[0] "_http_request_failure"
|
286
|
+
]
|
287
|
+
}
|
288
|
+
----
|
289
|
+
|
200
290
|
[id="plugins-{type}s-{plugin}-follow_redirects"]
|
201
291
|
===== `follow_redirects`
|
202
292
|
|
@@ -249,6 +339,14 @@ If you'd like to work with the request/response metadata.
|
|
249
339
|
Set this value to the name of the field you'd like to store a nested
|
250
340
|
hash of metadata.
|
251
341
|
|
342
|
+
[id="plugins-{type}s-{plugin}-password"]
|
343
|
+
===== `password`
|
344
|
+
|
345
|
+
* Value type is <<password,password>>
|
346
|
+
* There is no default value for this setting.
|
347
|
+
|
348
|
+
Password to be used in conjunction with <<plugins-{type}s-{plugin}-user>> for HTTP authentication.
|
349
|
+
|
252
350
|
[id="plugins-{type}s-{plugin}-pool_max"]
|
253
351
|
===== `pool_max`
|
254
352
|
|
@@ -316,6 +414,22 @@ See: rufus/scheduler for details about different schedule options and value stri
|
|
316
414
|
|
317
415
|
Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
318
416
|
|
417
|
+
[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
|
418
|
+
===== `ssl_verification_mode`
|
419
|
+
|
420
|
+
* Value type is <<string,string>>
|
421
|
+
* Supported values are: `full`, `none`
|
422
|
+
* Default value is `full`
|
423
|
+
|
424
|
+
Controls the verification of server certificates.
|
425
|
+
The `full` option verifies that the provided certificate is signed by a trusted authority (CA)
|
426
|
+
and also that the server’s hostname (or IP address) matches the names identified within the certificate.
|
427
|
+
|
428
|
+
The `none` setting performs no verification of the server’s certificate.
|
429
|
+
This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration.
|
430
|
+
It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors.
|
431
|
+
Using `none` in production environments is strongly discouraged.
|
432
|
+
|
319
433
|
[id="plugins-{type}s-{plugin}-target"]
|
320
434
|
===== `target`
|
321
435
|
|
@@ -324,6 +438,10 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
|
324
438
|
|
325
439
|
Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
|
326
440
|
|
441
|
+
TIP: When ECS is enabled, set `target` in the codec (if the codec has a `target` option).
|
442
|
+
Example: `codec => json { target => "TARGET_FIELD_NAME" }`
|
443
|
+
|
444
|
+
|
327
445
|
[id="plugins-{type}s-{plugin}-truststore"]
|
328
446
|
===== `truststore`
|
329
447
|
|
@@ -357,7 +475,45 @@ Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
|
357
475
|
* There is no default value for this setting.
|
358
476
|
|
359
477
|
A Hash of urls in this format : `"name" => "url"`.
|
360
|
-
The name and the url will be passed in the
|
478
|
+
The name and the url will be passed in the outputted event.
|
479
|
+
|
480
|
+
The values in urls can be either:
|
481
|
+
|
482
|
+
* a string url (which will be issued as an HTTP GET).
|
483
|
+
* a sub-hash containing many useful keys provided by the Manticore backend:
|
484
|
+
** url: the String url
|
485
|
+
** method: (optional) the HTTP method to use (defaults to GET)
|
486
|
+
** user: (optional) the HTTP Basic Auth user. The user must be under
|
487
|
+
an auth sub-hash for Manticore, but this plugin also accepts it either way.
|
488
|
+
** password: (optional) the HTTP Basic Auth password. The password
|
489
|
+
must be under an auth sub-hash for Manticore, but this plugin accepts it either way.
|
490
|
+
** headers: a hash containing key-value pairs of headers.
|
491
|
+
** body: a string (supported only on POST and PUT requests)
|
492
|
+
** possibly other options mentioned in the
|
493
|
+
https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs].
|
494
|
+
Note that Manticore options that are not explicitly documented above are not
|
495
|
+
thoroughly tested and therefore liable to break in unexpected ways if we
|
496
|
+
replace the backend.
|
497
|
+
|
498
|
+
*Notes:*
|
499
|
+
|
500
|
+
* Passwords specified as a part of `urls` are prone to exposure in plugin log output.
|
501
|
+
The plugin does not declare them as passwords, and therefore doesn't wrap them in
|
502
|
+
leak-reducing wrappers as we do elsewhere.
|
503
|
+
* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported
|
504
|
+
correctly. The strings `true` or `false` may get passed through, and in ruby any
|
505
|
+
string is "truthy."
|
506
|
+
* Our implementation of this plugin precludes the ability to specify auth[:eager]
|
507
|
+
as anything other than true
|
508
|
+
|
509
|
+
[id="plugins-{type}s-{plugin}-user"]
|
510
|
+
===== `user`
|
511
|
+
|
512
|
+
* Value type is <<string,string>>
|
513
|
+
* There is no default value for this setting.
|
514
|
+
|
515
|
+
Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
|
516
|
+
If you set this you must also set the <<plugins-{type}s-{plugin}-password>> option.
|
361
517
|
|
362
518
|
[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
|
363
519
|
===== `validate_after_inactivity`
|
@@ -5,9 +5,18 @@ require "logstash/plugin_mixins/http_client"
|
|
5
5
|
require "socket" # for Socket.gethostname
|
6
6
|
require "manticore"
|
7
7
|
require "rufus/scheduler"
|
8
|
+
require "logstash/plugin_mixins/ecs_compatibility_support"
|
9
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
|
10
|
+
require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
|
11
|
+
require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
8
12
|
|
9
13
|
class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
10
14
|
include LogStash::PluginMixins::HttpClient
|
15
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
16
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
|
17
|
+
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
18
|
+
|
19
|
+
extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
|
11
20
|
|
12
21
|
config_name "http_poller"
|
13
22
|
|
@@ -28,7 +37,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
28
37
|
config :schedule, :validate => :hash, :required => true
|
29
38
|
|
30
39
|
# Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
|
31
|
-
config :target, :validate => :
|
40
|
+
config :target, :validate => :field_reference
|
32
41
|
|
33
42
|
# If you'd like to work with the request/response metadata.
|
34
43
|
# Set this value to the name of the field you'd like to store a nested
|
@@ -42,12 +51,16 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
42
51
|
|
43
52
|
@logger.info("Registering http_poller Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
|
44
53
|
|
54
|
+
setup_ecs_field!
|
45
55
|
setup_requests!
|
46
56
|
end
|
47
57
|
|
58
|
+
# @overload
|
48
59
|
def stop
|
49
|
-
|
50
|
-
|
60
|
+
if @scheduler
|
61
|
+
@scheduler.shutdown # on newer Rufus (3.8) this joins on the scheduler thread
|
62
|
+
end
|
63
|
+
# TODO implement client.close as we as releasing it's pooled resources!
|
51
64
|
end
|
52
65
|
|
53
66
|
private
|
@@ -55,6 +68,35 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
55
68
|
@requests = Hash[@urls.map {|name, url| [name, normalize_request(url)] }]
|
56
69
|
end
|
57
70
|
|
71
|
+
private
|
72
|
+
# In the context of ECS, there are two type of events in this plugin, valid HTTP response and failure
|
73
|
+
# For a valid HTTP response, `url`, `request_method` and `host` are metadata of request.
|
74
|
+
# The call could retrieve event which contain `[url]`, `[http][request][method]`, `[host][hostname]` data
|
75
|
+
# Therefore, metadata should not write to those fields
|
76
|
+
# For a failure, `url`, `request_method` and `host` are primary data of the event because the plugin owns this event,
|
77
|
+
# so it writes to url.*, http.*, host.*
|
78
|
+
def setup_ecs_field!
|
79
|
+
@request_host_field = ecs_select[disabled: "[#{metadata_target}][host]", v1: "[#{metadata_target}][input][http_poller][request][host][hostname]"]
|
80
|
+
@response_code_field = ecs_select[disabled: "[#{metadata_target}][code]", v1: "[#{metadata_target}][input][http_poller][response][status_code]"]
|
81
|
+
@response_headers_field = ecs_select[disabled: "[#{metadata_target}][response_headers]", v1: "[#{metadata_target}][input][http_poller][response][headers]"]
|
82
|
+
@response_message_field = ecs_select[disabled: "[#{metadata_target}][response_message]", v1: "[#{metadata_target}][input][http_poller][response][status_message]"]
|
83
|
+
@response_time_s_field = ecs_select[disabled: "[#{metadata_target}][runtime_seconds]", v1: nil]
|
84
|
+
@response_time_ns_field = ecs_select[disabled: nil, v1: "[#{metadata_target}][input][http_poller][response][elapsed_time_ns]"]
|
85
|
+
@request_retry_count_field = ecs_select[disabled: "[#{metadata_target}][times_retried]", v1: "[#{metadata_target}][input][http_poller][request][retry_count]"]
|
86
|
+
@request_name_field = ecs_select[disabled: "[#{metadata_target}][name]", v1: "[#{metadata_target}][input][http_poller][request][name]"]
|
87
|
+
@original_request_field = ecs_select[disabled: "[#{metadata_target}][request]", v1: "[#{metadata_target}][input][http_poller][request][original]"]
|
88
|
+
|
89
|
+
@error_msg_field = ecs_select[disabled: "[http_request_failure][error]", v1: "[error][message]"]
|
90
|
+
@stack_trace_field = ecs_select[disabled: "[http_request_failure][backtrace]", v1: "[error][stack_trace]"]
|
91
|
+
@fail_original_request_field = ecs_select[disabled: "[http_request_failure][request]", v1: nil]
|
92
|
+
@fail_request_name_field = ecs_select[disabled: "[http_request_failure][name]", v1: nil]
|
93
|
+
@fail_response_time_s_field = ecs_select[disabled: "[http_request_failure][runtime_seconds]", v1: nil]
|
94
|
+
@fail_response_time_ns_field = ecs_select[disabled: nil, v1: "[event][duration]"]
|
95
|
+
@fail_request_url_field = ecs_select[disabled: nil, v1: "[url][full]"]
|
96
|
+
@fail_request_method_field = ecs_select[disabled: nil, v1: "[http][request][method]"]
|
97
|
+
@fail_request_host_field = ecs_select[disabled: nil, v1: "[host][hostname]"]
|
98
|
+
end
|
99
|
+
|
58
100
|
private
|
59
101
|
def normalize_request(url_or_spec)
|
60
102
|
if url_or_spec.is_a?(String)
|
@@ -124,16 +166,15 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
124
166
|
#schedule hash must contain exactly one of the allowed keys
|
125
167
|
msg_invalid_schedule = "Invalid config. schedule hash must contain " +
|
126
168
|
"exactly one of the following keys - cron, at, every or in"
|
127
|
-
raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
|
169
|
+
raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
|
128
170
|
schedule_type = @schedule.keys.first
|
129
171
|
schedule_value = @schedule[schedule_type]
|
130
172
|
raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
|
131
173
|
|
132
174
|
@scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
|
133
|
-
#as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
|
134
175
|
opts = schedule_type == "every" ? { :first_in => 0.01 } : {}
|
135
176
|
@scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
|
136
|
-
@scheduler.join
|
177
|
+
@scheduler.thread.join # due newer rufus (3.8) doing a blocking operation on scheduler.join
|
137
178
|
end
|
138
179
|
|
139
180
|
def run_once(queue)
|
@@ -151,10 +192,14 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
151
192
|
|
152
193
|
method, *request_opts = request
|
153
194
|
client.async.send(method, *request_opts).
|
154
|
-
on_success {|response| handle_success(queue, name, request, response, Time.now - started)}.
|
155
|
-
on_failure {|exception|
|
156
|
-
|
157
|
-
|
195
|
+
on_success {|response| handle_success(queue, name, request, response, Time.now - started) }.
|
196
|
+
on_failure {|exception| handle_failure(queue, name, request, exception, Time.now - started) }
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
# time diff in float to nanoseconds
|
201
|
+
def to_nanoseconds(time_diff)
|
202
|
+
(time_diff * 1000000).to_i
|
158
203
|
end
|
159
204
|
|
160
205
|
private
|
@@ -164,11 +209,11 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
164
209
|
# responses come up as "" which will cause the codec to not yield anything
|
165
210
|
if body && body.size > 0
|
166
211
|
decode_and_flush(@codec, body) do |decoded|
|
167
|
-
event = @target ?
|
212
|
+
event = @target ? targeted_event_factory.new_event(decoded.to_hash) : decoded
|
168
213
|
handle_decoded_event(queue, name, request, response, event, execution_time)
|
169
214
|
end
|
170
215
|
else
|
171
|
-
event =
|
216
|
+
event = event_factory.new_event
|
172
217
|
handle_decoded_event(queue, name, request, response, event, execution_time)
|
173
218
|
end
|
174
219
|
end
|
@@ -197,20 +242,10 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
197
242
|
private
|
198
243
|
# Beware, on old versions of manticore some uncommon failures are not handled
|
199
244
|
def handle_failure(queue, name, request, exception, execution_time)
|
200
|
-
event =
|
201
|
-
apply_metadata(event, name, request)
|
202
|
-
|
245
|
+
event = event_factory.new_event
|
203
246
|
event.tag("_http_request_failure")
|
204
|
-
|
205
|
-
|
206
|
-
# persisted by default, whereas metadata isn't. People don't like mysterious errors
|
207
|
-
event.set("http_request_failure", {
|
208
|
-
"request" => structure_request(request),
|
209
|
-
"name" => name,
|
210
|
-
"error" => exception.to_s,
|
211
|
-
"backtrace" => exception.backtrace,
|
212
|
-
"runtime_seconds" => execution_time
|
213
|
-
})
|
247
|
+
apply_metadata(event, name, request, nil, execution_time)
|
248
|
+
apply_failure_fields(event, name, request, exception, execution_time)
|
214
249
|
|
215
250
|
queue << event
|
216
251
|
rescue StandardError, java.lang.Exception => e
|
@@ -231,29 +266,39 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
231
266
|
end
|
232
267
|
|
233
268
|
private
|
234
|
-
def apply_metadata(event, name, request, response
|
269
|
+
def apply_metadata(event, name, request, response, execution_time)
|
235
270
|
return unless @metadata_target
|
236
|
-
event.set(@metadata_target, event_metadata(name, request, response, execution_time))
|
237
|
-
end
|
238
|
-
|
239
|
-
private
|
240
|
-
def event_metadata(name, request, response=nil, execution_time=nil)
|
241
|
-
m = {
|
242
|
-
"name" => name,
|
243
|
-
"host" => @host,
|
244
|
-
"request" => structure_request(request),
|
245
|
-
}
|
246
271
|
|
247
|
-
|
272
|
+
event.set(@request_host_field, @host)
|
273
|
+
event.set(@response_time_s_field, execution_time) if @response_time_s_field
|
274
|
+
event.set(@response_time_ns_field, to_nanoseconds(execution_time)) if @response_time_ns_field
|
275
|
+
event.set(@request_name_field, name)
|
276
|
+
event.set(@original_request_field, structure_request(request))
|
248
277
|
|
249
278
|
if response
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
279
|
+
event.set(@response_code_field, response.code)
|
280
|
+
event.set(@response_headers_field, response.headers)
|
281
|
+
event.set(@response_message_field, response.message)
|
282
|
+
event.set(@request_retry_count_field, response.times_retried)
|
254
283
|
end
|
284
|
+
end
|
255
285
|
|
256
|
-
|
286
|
+
private
|
287
|
+
def apply_failure_fields(event, name, request, exception, execution_time)
|
288
|
+
# This is also in the metadata, but we send it anyone because we want this
|
289
|
+
# persisted by default, whereas metadata isn't. People don't like mysterious errors
|
290
|
+
event.set(@fail_original_request_field, structure_request(request)) if @fail_original_request_field
|
291
|
+
event.set(@fail_request_name_field, name) if @fail_request_name_field
|
292
|
+
|
293
|
+
method, url, _ = request
|
294
|
+
event.set(@fail_request_url_field, url) if @fail_request_url_field
|
295
|
+
event.set(@fail_request_method_field, method.to_s) if @fail_request_method_field
|
296
|
+
event.set(@fail_request_host_field, @host) if @fail_request_host_field
|
297
|
+
|
298
|
+
event.set(@fail_response_time_s_field, execution_time) if @fail_response_time_s_field
|
299
|
+
event.set(@fail_response_time_ns_field, to_nanoseconds(execution_time)) if @fail_response_time_ns_field
|
300
|
+
event.set(@error_msg_field, exception.to_s)
|
301
|
+
event.set(@stack_trace_field, exception.backtrace)
|
257
302
|
end
|
258
303
|
|
259
304
|
private
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-http_poller'
|
3
|
-
s.version
|
3
|
+
s.version = '5.2.1'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Decodes the output of an HTTP API into events"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,9 +20,11 @@ Gem::Specification.new do |s|
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
|
-
s.add_runtime_dependency
|
24
|
-
s.add_runtime_dependency '
|
25
|
-
s.add_runtime_dependency '
|
23
|
+
s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0"
|
24
|
+
s.add_runtime_dependency 'rufus-scheduler', ">= 3.0.9"
|
25
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
|
26
|
+
s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0', '>= 1.0.1'
|
27
|
+
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|
26
28
|
|
27
29
|
s.add_development_dependency 'logstash-codec-json'
|
28
30
|
s.add_development_dependency 'logstash-codec-line'
|