logstash-input-http_poller 3.1.1 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/docs/index.asciidoc +400 -0
- data/lib/logstash/inputs/http_poller.rb +19 -72
- data/logstash-input-http_poller.gemspec +3 -3
- data/spec/inputs/http_poller_spec.rb +20 -4
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 09b540a6301e1a170482fffda921fbcc39c62b87
|
4
|
+
data.tar.gz: 7cb5b2d7a4693442739a74526180479c746ae6d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b84692ebad4fcc442cbbb5e7909c57f105cdaaa029ad2a9af467544e2b41407f798c3938b25f32429b4fa5d72996e6524b054bc589fd71a5b0cb491d9b83af7a
|
7
|
+
data.tar.gz: 7760092c26313343a5a994a87a44af5fc25c32de9636596e8c30d510a41882a03d4beda53a4a9aa8e7458ab4a8510f77cb0a8ddbd27d2d7562ef8b4108176ac0
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
## 3.3.0
|
2
|
+
- Add top level user/password options that apply to all URLs by default.
|
3
|
+
- Make user/password configurable per-URL nested at the top level without the extra auth hash
|
4
|
+
to make them more consistent with the global opts
|
5
|
+
- Bump mixin-http_client version
|
6
|
+
|
7
|
+
## 3.2.0
|
8
|
+
- Use eager auth. This means the client will send any credentials in its first request
|
9
|
+
rather than waiting for a 401 challenge
|
10
|
+
|
1
11
|
## 3.1.1
|
2
12
|
- Handle empty bodies correctly
|
3
13
|
## 3.1.0
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,400 @@
|
|
1
|
+
:plugin: http_poller
|
2
|
+
:type: input
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Http_poller
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and
|
24
|
+
send them on their merry way. The idea behind this plugins came from a need to read springboot
|
25
|
+
metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc.
|
26
|
+
|
27
|
+
==== Example
|
28
|
+
Reads from a list of urls and decodes the body of the response with a codec.
|
29
|
+
The config should look like this:
|
30
|
+
|
31
|
+
[source,ruby]
|
32
|
+
----------------------------------
|
33
|
+
input {
|
34
|
+
http_poller {
|
35
|
+
urls => {
|
36
|
+
test1 => "http://localhost:9200"
|
37
|
+
test2 => {
|
38
|
+
# Supports all options supported by ruby's Manticore HTTP client
|
39
|
+
method => get
|
40
|
+
user => "AzureDiamond"
|
41
|
+
password => "hunter2"
|
42
|
+
url => "http://localhost:9200/_cluster/health"
|
43
|
+
headers => {
|
44
|
+
Accept => "application/json"
|
45
|
+
}
|
46
|
+
}
|
47
|
+
}
|
48
|
+
request_timeout => 60
|
49
|
+
# Supports "cron", "every", "at" and "in" schedules by rufus scheduler
|
50
|
+
schedule => { cron => "* * * * * UTC"}
|
51
|
+
codec => "json"
|
52
|
+
# A hash of request metadata info (timing, response headers, etc.) will be sent here
|
53
|
+
metadata_target => "http_poller_metadata"
|
54
|
+
}
|
55
|
+
}
|
56
|
+
|
57
|
+
output {
|
58
|
+
stdout {
|
59
|
+
codec => rubydebug
|
60
|
+
}
|
61
|
+
}
|
62
|
+
----------------------------------
|
63
|
+
|
64
|
+
Using the HTTP poller with custom a custom CA or self signed cert.
|
65
|
+
|
66
|
+
If you have a self signed cert you will need to convert your server's certificate to a valid# `.jks` or `.p12` file. An easy way to do it is to run the following one-liner, substituting your server's URL for the placeholder `MYURL` and `MYPORT`.
|
67
|
+
|
68
|
+
[source,ruby]
|
69
|
+
----------------------------------
|
70
|
+
openssl s_client -showcerts -connect MYURL:MYPORT </dev/null 2>/dev/null|openssl x509 -outform PEM > downloaded_cert.pem; keytool -import -alias test -file downloaded_cert.pem -keystore downloaded_truststore.jks
|
71
|
+
----------------------------------
|
72
|
+
|
73
|
+
The above snippet will create two files `downloaded_cert.pem` and `downloaded_truststore.jks`. You will be prompted to set a password for the `jks` file during this process. To configure logstash use a config like the one that follows.
|
74
|
+
|
75
|
+
|
76
|
+
[source,ruby]
|
77
|
+
----------------------------------
|
78
|
+
http_poller {
|
79
|
+
urls => {
|
80
|
+
myurl => "https://myhostname:1234"
|
81
|
+
}
|
82
|
+
truststore => "/path/to/downloaded_truststore.jks"
|
83
|
+
truststore_password => "mypassword"
|
84
|
+
interval => 30
|
85
|
+
}
|
86
|
+
----------------------------------
|
87
|
+
|
88
|
+
|
89
|
+
[id="plugins-{type}s-{plugin}-options"]
|
90
|
+
==== Http_poller Input Configuration Options
|
91
|
+
|
92
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
|
93
|
+
|
94
|
+
[cols="<,<,<",options="header",]
|
95
|
+
|=======================================================================
|
96
|
+
|Setting |Input type|Required
|
97
|
+
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|no
|
98
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
99
|
+
| <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
|
100
|
+
| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
|
101
|
+
| <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
|
102
|
+
| <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
|
103
|
+
| <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
|
104
|
+
| <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
|
105
|
+
| <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
|
106
|
+
| <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
|
107
|
+
| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
|
108
|
+
| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
|
109
|
+
| <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
|
110
|
+
| <<plugins-{type}s-{plugin}-metadata_target>> |<<string,string>>|No
|
111
|
+
| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
|
112
|
+
| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
|
113
|
+
| <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
|
114
|
+
| <<plugins-{type}s-{plugin}-request_timeout>> |<<number,number>>|No
|
115
|
+
| <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
|
116
|
+
| <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|No
|
117
|
+
| <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
|
118
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_validation>> |<<boolean,boolean>>|No
|
119
|
+
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
120
|
+
| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
|
121
|
+
| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
|
122
|
+
| <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
|
123
|
+
| <<plugins-{type}s-{plugin}-urls>> |<<hash,hash>>|Yes
|
124
|
+
| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
|
125
|
+
|=======================================================================
|
126
|
+
|
127
|
+
Also see <<plugins-{type}s-common-options>> for a list of options supported by all
|
128
|
+
input plugins.
|
129
|
+
|
130
|
+
|
131
|
+
|
132
|
+
[id="plugins-{type}s-{plugin}-user"]
|
133
|
+
===== `user`
|
134
|
+
|
135
|
+
* Value type is <<string,string>>
|
136
|
+
* There is no default value for this setting.
|
137
|
+
|
138
|
+
Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
|
139
|
+
If you set this you must also set the `password` option.
|
140
|
+
|
141
|
+
[id="plugins-{type}s-{plugin}-password"]
|
142
|
+
===== `password`
|
143
|
+
|
144
|
+
* Value type is <<password,password>>
|
145
|
+
* There is no default value for this setting.
|
146
|
+
|
147
|
+
Password to be used in conjunction with the username for HTTP authentication.
|
148
|
+
|
149
|
+
[id="plugins-{type}s-{plugin}-automatic_retries"]
|
150
|
+
===== `automatic_retries`
|
151
|
+
|
152
|
+
* Value type is <<number,number>>
|
153
|
+
* Default value is `1`
|
154
|
+
|
155
|
+
How many times should the client retry a failing URL. We highly recommend NOT setting this value
|
156
|
+
to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
|
157
|
+
Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
|
158
|
+
|
159
|
+
[id="plugins-{type}s-{plugin}-cacert"]
|
160
|
+
===== `cacert`
|
161
|
+
|
162
|
+
* Value type is <<path,path>>
|
163
|
+
* There is no default value for this setting.
|
164
|
+
|
165
|
+
If you need to use a custom X.509 CA (.pem certs) specify the path to that here
|
166
|
+
|
167
|
+
[id="plugins-{type}s-{plugin}-client_cert"]
|
168
|
+
===== `client_cert`
|
169
|
+
|
170
|
+
* Value type is <<path,path>>
|
171
|
+
* There is no default value for this setting.
|
172
|
+
|
173
|
+
If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
|
174
|
+
|
175
|
+
[id="plugins-{type}s-{plugin}-client_key"]
|
176
|
+
===== `client_key`
|
177
|
+
|
178
|
+
* Value type is <<path,path>>
|
179
|
+
* There is no default value for this setting.
|
180
|
+
|
181
|
+
If you're using a client certificate specify the path to the encryption key here
|
182
|
+
|
183
|
+
[id="plugins-{type}s-{plugin}-connect_timeout"]
|
184
|
+
===== `connect_timeout`
|
185
|
+
|
186
|
+
* Value type is <<number,number>>
|
187
|
+
* Default value is `10`
|
188
|
+
|
189
|
+
Timeout (in seconds) to wait for a connection to be established. Default is `10s`
|
190
|
+
|
191
|
+
[id="plugins-{type}s-{plugin}-cookies"]
|
192
|
+
===== `cookies`
|
193
|
+
|
194
|
+
* Value type is <<boolean,boolean>>
|
195
|
+
* Default value is `true`
|
196
|
+
|
197
|
+
Enable cookie support. With this enabled the client will persist cookies
|
198
|
+
across requests as a normal web browser would. Enabled by default
|
199
|
+
|
200
|
+
[id="plugins-{type}s-{plugin}-follow_redirects"]
|
201
|
+
===== `follow_redirects`
|
202
|
+
|
203
|
+
* Value type is <<boolean,boolean>>
|
204
|
+
* Default value is `true`
|
205
|
+
|
206
|
+
Should redirects be followed? Defaults to `true`
|
207
|
+
|
208
|
+
[id="plugins-{type}s-{plugin}-interval"]
|
209
|
+
===== `interval` (DEPRECATED)
|
210
|
+
|
211
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
212
|
+
* Value type is <<number,number>>
|
213
|
+
* There is no default value for this setting.
|
214
|
+
|
215
|
+
How often (in seconds) the urls will be called
|
216
|
+
DEPRECATED. Use 'schedule' option instead.
|
217
|
+
If both interval and schedule options are specified, interval
|
218
|
+
option takes higher precedence
|
219
|
+
|
220
|
+
[id="plugins-{type}s-{plugin}-keepalive"]
|
221
|
+
===== `keepalive`
|
222
|
+
|
223
|
+
* Value type is <<boolean,boolean>>
|
224
|
+
* Default value is `true`
|
225
|
+
|
226
|
+
Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
|
227
|
+
one with this to fix interactions with broken keepalive implementations.
|
228
|
+
|
229
|
+
[id="plugins-{type}s-{plugin}-keystore"]
|
230
|
+
===== `keystore`
|
231
|
+
|
232
|
+
* Value type is <<path,path>>
|
233
|
+
* There is no default value for this setting.
|
234
|
+
|
235
|
+
If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
|
236
|
+
|
237
|
+
[id="plugins-{type}s-{plugin}-keystore_password"]
|
238
|
+
===== `keystore_password`
|
239
|
+
|
240
|
+
* Value type is <<password,password>>
|
241
|
+
* There is no default value for this setting.
|
242
|
+
|
243
|
+
Specify the keystore password here.
|
244
|
+
Note, most .jks files created with keytool require a password!
|
245
|
+
|
246
|
+
[id="plugins-{type}s-{plugin}-keystore_type"]
|
247
|
+
===== `keystore_type`
|
248
|
+
|
249
|
+
* Value type is <<string,string>>
|
250
|
+
* Default value is `"JKS"`
|
251
|
+
|
252
|
+
Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
253
|
+
|
254
|
+
[id="plugins-{type}s-{plugin}-metadata_target"]
|
255
|
+
===== `metadata_target`
|
256
|
+
|
257
|
+
* Value type is <<string,string>>
|
258
|
+
* Default value is `"@metadata"`
|
259
|
+
|
260
|
+
If you'd like to work with the request/response metadata.
|
261
|
+
Set this value to the name of the field you'd like to store a nested
|
262
|
+
hash of metadata.
|
263
|
+
|
264
|
+
[id="plugins-{type}s-{plugin}-pool_max"]
|
265
|
+
===== `pool_max`
|
266
|
+
|
267
|
+
* Value type is <<number,number>>
|
268
|
+
* Default value is `50`
|
269
|
+
|
270
|
+
Max number of concurrent connections. Defaults to `50`
|
271
|
+
|
272
|
+
[id="plugins-{type}s-{plugin}-pool_max_per_route"]
|
273
|
+
===== `pool_max_per_route`
|
274
|
+
|
275
|
+
* Value type is <<number,number>>
|
276
|
+
* Default value is `25`
|
277
|
+
|
278
|
+
Max number of concurrent connections to a single host. Defaults to `25`
|
279
|
+
|
280
|
+
[id="plugins-{type}s-{plugin}-proxy"]
|
281
|
+
===== `proxy`
|
282
|
+
|
283
|
+
* Value type is <<string,string>>
|
284
|
+
* There is no default value for this setting.
|
285
|
+
|
286
|
+
If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
|
287
|
+
|
288
|
+
1. Proxy host in form: `http://proxy.org:1234`
|
289
|
+
2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
|
290
|
+
3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}`
|
291
|
+
|
292
|
+
[id="plugins-{type}s-{plugin}-request_timeout"]
|
293
|
+
===== `request_timeout`
|
294
|
+
|
295
|
+
* Value type is <<number,number>>
|
296
|
+
* Default value is `60`
|
297
|
+
|
298
|
+
This module makes it easy to add a very fully configured HTTP client to logstash
|
299
|
+
based on [Manticore](https://github.com/cheald/manticore).
|
300
|
+
For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
|
301
|
+
Timeout (in seconds) for the entire request
|
302
|
+
|
303
|
+
[id="plugins-{type}s-{plugin}-retry_non_idempotent"]
|
304
|
+
===== `retry_non_idempotent`
|
305
|
+
|
306
|
+
* Value type is <<boolean,boolean>>
|
307
|
+
* Default value is `false`
|
308
|
+
|
309
|
+
If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
|
310
|
+
|
311
|
+
[id="plugins-{type}s-{plugin}-schedule"]
|
312
|
+
===== `schedule`
|
313
|
+
|
314
|
+
* Value type is <<hash,hash>>
|
315
|
+
* There is no default value for this setting.
|
316
|
+
|
317
|
+
Schedule of when to periodically poll from the urls
|
318
|
+
Format: A hash with
|
319
|
+
+ key: "cron" | "every" | "in" | "at"
|
320
|
+
+ value: string
|
321
|
+
Examples:
|
322
|
+
a) { "every" => "1h" }
|
323
|
+
b) { "cron" => "* * * * * UTC" }
|
324
|
+
See: rufus/scheduler for details about different schedule options and value string format
|
325
|
+
|
326
|
+
[id="plugins-{type}s-{plugin}-socket_timeout"]
|
327
|
+
===== `socket_timeout`
|
328
|
+
|
329
|
+
* Value type is <<number,number>>
|
330
|
+
* Default value is `10`
|
331
|
+
|
332
|
+
Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
333
|
+
|
334
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_validation"]
|
335
|
+
===== `ssl_certificate_validation`
|
336
|
+
|
337
|
+
* Value type is <<boolean,boolean>>
|
338
|
+
* Default value is `true`
|
339
|
+
|
340
|
+
Set this to false to disable SSL/TLS certificate validation
|
341
|
+
Note: setting this to false is generally considered insecure!
|
342
|
+
|
343
|
+
[id="plugins-{type}s-{plugin}-target"]
|
344
|
+
===== `target`
|
345
|
+
|
346
|
+
* Value type is <<string,string>>
|
347
|
+
* There is no default value for this setting.
|
348
|
+
|
349
|
+
Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
|
350
|
+
|
351
|
+
[id="plugins-{type}s-{plugin}-truststore"]
|
352
|
+
===== `truststore`
|
353
|
+
|
354
|
+
* Value type is <<path,path>>
|
355
|
+
* There is no default value for this setting.
|
356
|
+
|
357
|
+
If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
|
358
|
+
|
359
|
+
[id="plugins-{type}s-{plugin}-truststore_password"]
|
360
|
+
===== `truststore_password`
|
361
|
+
|
362
|
+
* Value type is <<password,password>>
|
363
|
+
* There is no default value for this setting.
|
364
|
+
|
365
|
+
Specify the truststore password here.
|
366
|
+
Note, most .jks files created with keytool require a password!
|
367
|
+
|
368
|
+
[id="plugins-{type}s-{plugin}-truststore_type"]
|
369
|
+
===== `truststore_type`
|
370
|
+
|
371
|
+
* Value type is <<string,string>>
|
372
|
+
* Default value is `"JKS"`
|
373
|
+
|
374
|
+
Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
375
|
+
|
376
|
+
[id="plugins-{type}s-{plugin}-urls"]
|
377
|
+
===== `urls`
|
378
|
+
|
379
|
+
* This is a required setting.
|
380
|
+
* Value type is <<hash,hash>>
|
381
|
+
* There is no default value for this setting.
|
382
|
+
|
383
|
+
A Hash of urls in this format : `"name" => "url"`.
|
384
|
+
The name and the url will be passed in the outputed event
|
385
|
+
|
386
|
+
[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
|
387
|
+
===== `validate_after_inactivity`
|
388
|
+
|
389
|
+
* Value type is <<number,number>>
|
390
|
+
* Default value is `200`
|
391
|
+
|
392
|
+
How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
393
|
+
# You may want to set this lower, possibly to 0 if you get connection errors regularly
|
394
|
+
Quoting the Apache commons docs (this client is based Apache Commmons):
|
395
|
+
'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
|
396
|
+
See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
397
|
+
|
398
|
+
|
399
|
+
|
400
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -6,76 +6,6 @@ require "socket" # for Socket.gethostname
|
|
6
6
|
require "manticore"
|
7
7
|
require "rufus/scheduler"
|
8
8
|
|
9
|
-
# This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and
|
10
|
-
# send them on their merry way. The idea behind this plugins came from a need to read springboot
|
11
|
-
# metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc.
|
12
|
-
#
|
13
|
-
# ==== Example
|
14
|
-
# Reads from a list of urls and decodes the body of the response with a codec.
|
15
|
-
# The config should look like this:
|
16
|
-
#
|
17
|
-
# [source,ruby]
|
18
|
-
# ----------------------------------
|
19
|
-
# input {
|
20
|
-
# http_poller {
|
21
|
-
# urls => {
|
22
|
-
# test1 => "http://localhost:9200"
|
23
|
-
# test2 => {
|
24
|
-
# # Supports all options supported by ruby's Manticore HTTP client
|
25
|
-
# method => get
|
26
|
-
# url => "http://localhost:9200/_cluster/health"
|
27
|
-
# headers => {
|
28
|
-
# Accept => "application/json"
|
29
|
-
# }
|
30
|
-
# auth => {
|
31
|
-
# user => "AzureDiamond"
|
32
|
-
# password => "hunter2"
|
33
|
-
# }
|
34
|
-
# }
|
35
|
-
# }
|
36
|
-
# request_timeout => 60
|
37
|
-
# # Supports "cron", "every", "at" and "in" schedules by rufus scheduler
|
38
|
-
# schedule => { cron => "* * * * * UTC"}
|
39
|
-
# codec => "json"
|
40
|
-
# # A hash of request metadata info (timing, response headers, etc.) will be sent here
|
41
|
-
# metadata_target => "http_poller_metadata"
|
42
|
-
# }
|
43
|
-
# }
|
44
|
-
#
|
45
|
-
# output {
|
46
|
-
# stdout {
|
47
|
-
# codec => rubydebug
|
48
|
-
# }
|
49
|
-
# }
|
50
|
-
# ----------------------------------
|
51
|
-
#
|
52
|
-
# Using the HTTP poller with custom a custom CA or self signed cert.
|
53
|
-
#
|
54
|
-
# If you have a self signed cert you will need to convert your server's certificate to a valid# `.jks` or `.p12` file. An easy way to do it is to run the following one-liner, substituting your server's URL for the placeholder `MYURL` and `MYPORT`.
|
55
|
-
#
|
56
|
-
# [source,ruby]
|
57
|
-
# ----------------------------------
|
58
|
-
# openssl s_client -showcerts -connect MYURL:MYPORT </dev/null 2>/dev/null|openssl x509 -outform PEM > downloaded_cert.pem; keytool -import -alias test -file downloaded_cert.pem -keystore downloaded_truststore.jks
|
59
|
-
# ----------------------------------
|
60
|
-
#
|
61
|
-
# The above snippet will create two files `downloaded_cert.pem` and `downloaded_truststore.jks`. You will be prompted to set a password for the `jks` file during this process. To configure logstash use a config like the one that follows.
|
62
|
-
#
|
63
|
-
#
|
64
|
-
# [source,ruby]
|
65
|
-
# ----------------------------------
|
66
|
-
#input {
|
67
|
-
# http_poller {
|
68
|
-
# urls => {
|
69
|
-
# myurl => "https://myhostname:1234"
|
70
|
-
# }
|
71
|
-
# truststore => "/path/to/downloaded_truststore.jks"
|
72
|
-
# truststore_password => "mypassword"
|
73
|
-
# interval => 30
|
74
|
-
# }
|
75
|
-
#}
|
76
|
-
# ----------------------------------
|
77
|
-
#
|
78
|
-
|
79
9
|
class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
80
10
|
include LogStash::PluginMixins::HttpClient
|
81
11
|
|
@@ -144,9 +74,26 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
144
74
|
method = (spec.delete(:method) || :get).to_sym.downcase
|
145
75
|
url = spec.delete(:url)
|
146
76
|
|
147
|
-
#
|
148
|
-
|
77
|
+
# Manticore wants auth options that are like {:auth => {:user => u, :pass => p}}
|
78
|
+
# We allow that because earlier versions of this plugin documented that as the main way to
|
79
|
+
# to do things, but now prefer top level "user", and "password" options
|
80
|
+
# So, if the top level user/password are defined they are moved to the :auth key for manticore
|
81
|
+
# if those attributes are already in :auth they still need to be transformed to symbols
|
82
|
+
auth = spec[:auth]
|
83
|
+
user = spec.delete(:user) || (auth && auth["user"])
|
84
|
+
password = spec.delete(:password) || (auth && auth["password"])
|
85
|
+
|
86
|
+
if user.nil? ^ password.nil?
|
87
|
+
raise LogStash::ConfigurationError, "'user' and 'password' must both be specified for input HTTP poller!"
|
88
|
+
end
|
149
89
|
|
90
|
+
if user && password
|
91
|
+
spec[:auth] = {
|
92
|
+
user: user,
|
93
|
+
pass: password,
|
94
|
+
eager: true
|
95
|
+
}
|
96
|
+
end
|
150
97
|
res = [method, url, spec]
|
151
98
|
else
|
152
99
|
raise LogStash::ConfigurationError, "Invalid URL or request spec: '#{url_or_spec}', expected a String or Hash!"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-http_poller'
|
3
|
-
s.version = '3.
|
3
|
+
s.version = '3.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Poll HTTP endpoints with Logstash."
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files = Dir[
|
13
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
14
14
|
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
|
-
s.add_runtime_dependency 'logstash-mixin-http_client', ">=
|
23
|
+
s.add_runtime_dependency 'logstash-mixin-http_client', ">= 5.0.0", "< 6.0.0"
|
24
24
|
s.add_runtime_dependency 'stud', "~> 0.0.22"
|
25
25
|
s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
|
26
26
|
|
@@ -106,9 +106,7 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
109
|
-
|
110
|
-
let(:url) { {"url" => "http://localhost", "method" => "get", "auth" => auth} }
|
111
|
-
|
109
|
+
shared_examples "auth" do
|
112
110
|
context "with auth enabled but no pass" do
|
113
111
|
let(:auth) { {"user" => "foo"} }
|
114
112
|
|
@@ -131,10 +129,28 @@ describe LogStash::Inputs::HTTP_Poller do
|
|
131
129
|
end
|
132
130
|
|
133
131
|
it "should properly set the auth parameter" do
|
134
|
-
expect(normalized[2][:auth]).to
|
132
|
+
expect(normalized[2][:auth]).to eq({
|
133
|
+
:user => auth["user"],
|
134
|
+
:pass => auth["password"],
|
135
|
+
:eager => true
|
136
|
+
})
|
135
137
|
end
|
136
138
|
end
|
137
139
|
end
|
140
|
+
|
141
|
+
# Legacy way of doing things, kept for backwards compat.
|
142
|
+
describe "auth with nested auth hash" do
|
143
|
+
let(:url) { {"url" => "http://localhost", "method" => "get", "auth" => auth} }
|
144
|
+
|
145
|
+
include_examples("auth")
|
146
|
+
end
|
147
|
+
|
148
|
+
# The new 'right' way to do things
|
149
|
+
describe "auth with direct auth options" do
|
150
|
+
let(:url) { {"url" => "http://localhost", "method" => "get", "user" => auth["user"], "password" => auth["password"]} }
|
151
|
+
|
152
|
+
include_examples("auth")
|
153
|
+
end
|
138
154
|
end
|
139
155
|
end
|
140
156
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-http_poller
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-05-08 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,10 +50,10 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
53
|
+
version: 5.0.0
|
54
54
|
- - "<"
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version:
|
56
|
+
version: 6.0.0
|
57
57
|
name: logstash-mixin-http_client
|
58
58
|
prerelease: false
|
59
59
|
type: :runtime
|
@@ -61,10 +61,10 @@ dependencies:
|
|
61
61
|
requirements:
|
62
62
|
- - ">="
|
63
63
|
- !ruby/object:Gem::Version
|
64
|
-
version:
|
64
|
+
version: 5.0.0
|
65
65
|
- - "<"
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version:
|
67
|
+
version: 6.0.0
|
68
68
|
- !ruby/object:Gem::Dependency
|
69
69
|
requirement: !ruby/object:Gem::Requirement
|
70
70
|
requirements:
|
@@ -160,6 +160,7 @@ files:
|
|
160
160
|
- Gemfile
|
161
161
|
- LICENSE
|
162
162
|
- README.md
|
163
|
+
- docs/index.asciidoc
|
163
164
|
- lib/logstash/inputs/http_poller.rb
|
164
165
|
- logstash-input-http_poller.gemspec
|
165
166
|
- spec/inputs/http_poller_spec.rb
|