logstash-output-httpcodec 5.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +101 -0
- data/CONTRIBUTORS +17 -0
- data/Gemfile +11 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +98 -0
- data/docs/index.asciidoc +391 -0
- data/lib/logstash/outputs/httpcodec.rb +390 -0
- data/logstash-output-httpcodec.gemspec +28 -0
- data/spec/outputs/httpcodec_spec.rb +364 -0
- data/spec/supports/compressed_requests.rb +38 -0
- metadata +142 -0
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,391 @@
|
|
1
|
+
:plugin: http
|
2
|
+
:type: output
|
3
|
+
:default_codec: plain
|
4
|
+
|
5
|
+
///////////////////////////////////////////
|
6
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
7
|
+
///////////////////////////////////////////
|
8
|
+
:version: %VERSION%
|
9
|
+
:release_date: %RELEASE_DATE%
|
10
|
+
:changelog_url: %CHANGELOG_URL%
|
11
|
+
:include_path: ../../../../logstash/docs/include
|
12
|
+
///////////////////////////////////////////
|
13
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
14
|
+
///////////////////////////////////////////
|
15
|
+
|
16
|
+
[id="plugins-{type}s-{plugin}"]
|
17
|
+
|
18
|
+
=== Http output plugin
|
19
|
+
|
20
|
+
include::{include_path}/plugin_header.asciidoc[]
|
21
|
+
|
22
|
+
==== Description
|
23
|
+
|
24
|
+
This output lets you send events to a generic HTTP(S) endpoint.
|
25
|
+
|
26
|
+
This output will execute up to 'pool_max' requests in parallel for performance.
|
27
|
+
Consider this when tuning this plugin for performance.
|
28
|
+
|
29
|
+
Additionally, note that when parallel execution is used strict ordering of events is not
|
30
|
+
guaranteed!
|
31
|
+
|
32
|
+
Beware, to use codecs, please use the 'format' option for now.
|
33
|
+
|
34
|
+
[id="plugins-{type}s-{plugin}-options"]
|
35
|
+
==== Http Output Configuration Options
|
36
|
+
|
37
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
38
|
+
|
39
|
+
[cols="<,<,<",options="header",]
|
40
|
+
|=======================================================================
|
41
|
+
|Setting |Input type|Required
|
42
|
+
| <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
|
43
|
+
| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
|
44
|
+
| <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
|
45
|
+
| <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
|
46
|
+
| <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
|
47
|
+
| <<plugins-{type}s-{plugin}-content_type>> |<<string,string>>|No
|
48
|
+
| <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
|
49
|
+
| <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
|
50
|
+
| <<plugins-{type}s-{plugin}-format>> |<<string,string>>, one of `["json", "json_batch", "form", "message"]`|No
|
51
|
+
| <<plugins-{type}s-{plugin}-headers>> |<<hash,hash>>|No
|
52
|
+
| <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
|
53
|
+
| <<plugins-{type}s-{plugin}-http_method>> |<<string,string>>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes
|
54
|
+
| <<plugins-{type}s-{plugin}-ignorable_codes>> |<<number,number>>|No
|
55
|
+
| <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
|
56
|
+
| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
|
57
|
+
| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
|
58
|
+
| <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
|
59
|
+
| <<plugins-{type}s-{plugin}-mapping>> |<<hash,hash>>|No
|
60
|
+
| <<plugins-{type}s-{plugin}-message>> |<<string,string>>|No
|
61
|
+
| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
|
62
|
+
| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
|
63
|
+
| <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
|
64
|
+
| <<plugins-{type}s-{plugin}-request_timeout>> |<<number,number>>|No
|
65
|
+
| <<plugins-{type}s-{plugin}-retry_failed>> |<<boolean,boolean>>|No
|
66
|
+
| <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
|
67
|
+
| <<plugins-{type}s-{plugin}-retryable_codes>> |<<number,number>>|No
|
68
|
+
| <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
|
69
|
+
| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
|
70
|
+
| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
|
71
|
+
| <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
|
72
|
+
| <<plugins-{type}s-{plugin}-url>> |<<string,string>>|Yes
|
73
|
+
| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
|
74
|
+
|=======================================================================
|
75
|
+
|
76
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
77
|
+
output plugins.
|
78
|
+
|
79
|
+
|
80
|
+
|
81
|
+
[id="plugins-{type}s-{plugin}-automatic_retries"]
|
82
|
+
===== `automatic_retries`
|
83
|
+
|
84
|
+
* Value type is <<number,number>>
|
85
|
+
* Default value is `1`
|
86
|
+
|
87
|
+
How many times should the client retry a failing URL. We highly recommend NOT setting this value
|
88
|
+
to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
|
89
|
+
Only IO related failures will be retried, such as connection timeouts and unreachable hosts.
|
90
|
+
Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting,
|
91
|
+
unless `retry_failed` is set.
|
92
|
+
Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
|
93
|
+
|
94
|
+
[id="plugins-{type}s-{plugin}-cacert"]
|
95
|
+
===== `cacert`
|
96
|
+
|
97
|
+
* Value type is <<path,path>>
|
98
|
+
* There is no default value for this setting.
|
99
|
+
|
100
|
+
If you need to use a custom X.509 CA (.pem certs) specify the path to that here
|
101
|
+
|
102
|
+
[id="plugins-{type}s-{plugin}-client_cert"]
|
103
|
+
===== `client_cert`
|
104
|
+
|
105
|
+
* Value type is <<path,path>>
|
106
|
+
* There is no default value for this setting.
|
107
|
+
|
108
|
+
If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
|
109
|
+
|
110
|
+
[id="plugins-{type}s-{plugin}-client_key"]
|
111
|
+
===== `client_key`
|
112
|
+
|
113
|
+
* Value type is <<path,path>>
|
114
|
+
* There is no default value for this setting.
|
115
|
+
|
116
|
+
If you're using a client certificate specify the path to the encryption key here
|
117
|
+
|
118
|
+
[id="plugins-{type}s-{plugin}-connect_timeout"]
|
119
|
+
===== `connect_timeout`
|
120
|
+
|
121
|
+
* Value type is <<number,number>>
|
122
|
+
* Default value is `10`
|
123
|
+
|
124
|
+
Timeout (in seconds) to wait for a connection to be established. Default is `10s`
|
125
|
+
|
126
|
+
[id="plugins-{type}s-{plugin}-content_type"]
|
127
|
+
===== `content_type`
|
128
|
+
|
129
|
+
* Value type is <<string,string>>
|
130
|
+
* There is no default value for this setting.
|
131
|
+
|
132
|
+
Content type
|
133
|
+
|
134
|
+
If not specified, this defaults to the following:
|
135
|
+
|
136
|
+
* if format is "json", "application/json"
|
137
|
+
* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request.
|
138
|
+
* if format is "form", "application/x-www-form-urlencoded"
|
139
|
+
|
140
|
+
[id="plugins-{type}s-{plugin}-cookies"]
|
141
|
+
===== `cookies`
|
142
|
+
|
143
|
+
* Value type is <<boolean,boolean>>
|
144
|
+
* Default value is `true`
|
145
|
+
|
146
|
+
Enable cookie support. With this enabled the client will persist cookies
|
147
|
+
across requests as a normal web browser would. Enabled by default
|
148
|
+
|
149
|
+
[id="plugins-{type}s-{plugin}-follow_redirects"]
|
150
|
+
===== `follow_redirects`
|
151
|
+
|
152
|
+
* Value type is <<boolean,boolean>>
|
153
|
+
* Default value is `true`
|
154
|
+
|
155
|
+
Should redirects be followed? Defaults to `true`
|
156
|
+
|
157
|
+
[id="plugins-{type}s-{plugin}-format"]
|
158
|
+
===== `format`
|
159
|
+
|
160
|
+
* Value can be any of: `json`, `json_batch`, `form`, `message`, `codec`
|
161
|
+
* Default value is `"json"`
|
162
|
+
|
163
|
+
Set the format of the http body.
|
164
|
+
|
165
|
+
If json_batch, each batch of events received by this output will be placed
|
166
|
+
into a single JSON array and sent in one request. This is particularly useful
|
167
|
+
for high throughput scenarios such as sending data between Logstash instaces.
|
168
|
+
|
169
|
+
If form, then the body will be the mapping (or whole event) converted
|
170
|
+
into a query parameter string, e.g. `foo=bar&baz=fizz...`
|
171
|
+
|
172
|
+
If message, then the body will be the result of formatting the event according to message
|
173
|
+
|
174
|
+
Otherwise, the event is sent as json.
|
175
|
+
|
176
|
+
[id="plugins-{type}s-{plugin}-headers"]
|
177
|
+
===== `headers`
|
178
|
+
|
179
|
+
* Value type is <<hash,hash>>
|
180
|
+
* There is no default value for this setting.
|
181
|
+
|
182
|
+
Custom headers to use
|
183
|
+
format is `headers => ["X-My-Header", "%{host}"]`
|
184
|
+
|
185
|
+
[id="plugins-{type}s-{plugin}-http_compression"]
|
186
|
+
===== `http_compression`
|
187
|
+
|
188
|
+
* Value type is <<boolean,boolean>>
|
189
|
+
* Default value is `false`
|
190
|
+
|
191
|
+
Enable request compression support. With this enabled the plugin will compress
|
192
|
+
http requests using gzip.
|
193
|
+
|
194
|
+
[id="plugins-{type}s-{plugin}-http_method"]
|
195
|
+
===== `http_method`
|
196
|
+
|
197
|
+
* This is a required setting.
|
198
|
+
* Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head`
|
199
|
+
* There is no default value for this setting.
|
200
|
+
|
201
|
+
The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
|
202
|
+
|
203
|
+
[id="plugins-{type}s-{plugin}-ignorable_codes"]
|
204
|
+
===== `ignorable_codes`
|
205
|
+
|
206
|
+
* Value type is <<number,number>>
|
207
|
+
* There is no default value for this setting.
|
208
|
+
|
209
|
+
If you would like to consider some non-2xx codes to be successes
|
210
|
+
enumerate them here. Responses returning these codes will be considered successes
|
211
|
+
|
212
|
+
[id="plugins-{type}s-{plugin}-keepalive"]
|
213
|
+
===== `keepalive`
|
214
|
+
|
215
|
+
* Value type is <<boolean,boolean>>
|
216
|
+
* Default value is `true`
|
217
|
+
|
218
|
+
Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
|
219
|
+
one with this to fix interactions with broken keepalive implementations.
|
220
|
+
|
221
|
+
[id="plugins-{type}s-{plugin}-keystore"]
|
222
|
+
===== `keystore`
|
223
|
+
|
224
|
+
* Value type is <<path,path>>
|
225
|
+
* There is no default value for this setting.
|
226
|
+
|
227
|
+
If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
|
228
|
+
|
229
|
+
[id="plugins-{type}s-{plugin}-keystore_password"]
|
230
|
+
===== `keystore_password`
|
231
|
+
|
232
|
+
* Value type is <<password,password>>
|
233
|
+
* There is no default value for this setting.
|
234
|
+
|
235
|
+
Specify the keystore password here.
|
236
|
+
Note, most .jks files created with keytool require a password!
|
237
|
+
|
238
|
+
[id="plugins-{type}s-{plugin}-keystore_type"]
|
239
|
+
===== `keystore_type`
|
240
|
+
|
241
|
+
* Value type is <<string,string>>
|
242
|
+
* Default value is `"JKS"`
|
243
|
+
|
244
|
+
Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
245
|
+
|
246
|
+
[id="plugins-{type}s-{plugin}-mapping"]
|
247
|
+
===== `mapping`
|
248
|
+
|
249
|
+
* Value type is <<hash,hash>>
|
250
|
+
* There is no default value for this setting.
|
251
|
+
|
252
|
+
This lets you choose the structure and parts of the event that are sent.
|
253
|
+
|
254
|
+
|
255
|
+
For example:
|
256
|
+
[source,ruby]
|
257
|
+
mapping => {"foo" => "%{host}"
|
258
|
+
"bar" => "%{type}"}
|
259
|
+
|
260
|
+
[id="plugins-{type}s-{plugin}-message"]
|
261
|
+
===== `message`
|
262
|
+
|
263
|
+
* Value type is <<string,string>>
|
264
|
+
* There is no default value for this setting.
|
265
|
+
|
266
|
+
|
267
|
+
|
268
|
+
[id="plugins-{type}s-{plugin}-pool_max"]
|
269
|
+
===== `pool_max`
|
270
|
+
|
271
|
+
* Value type is <<number,number>>
|
272
|
+
* Default value is `50`
|
273
|
+
|
274
|
+
Max number of concurrent connections. Defaults to `50`
|
275
|
+
|
276
|
+
[id="plugins-{type}s-{plugin}-pool_max_per_route"]
|
277
|
+
===== `pool_max_per_route`
|
278
|
+
|
279
|
+
* Value type is <<number,number>>
|
280
|
+
* Default value is `25`
|
281
|
+
|
282
|
+
Max number of concurrent connections to a single host. Defaults to `25`
|
283
|
+
|
284
|
+
[id="plugins-{type}s-{plugin}-proxy"]
|
285
|
+
===== `proxy`
|
286
|
+
|
287
|
+
* Value type is <<string,string>>
|
288
|
+
* There is no default value for this setting.
|
289
|
+
|
290
|
+
If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
|
291
|
+
|
292
|
+
1. Proxy host in form: `http://proxy.org:1234`
|
293
|
+
2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
|
294
|
+
3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}`
|
295
|
+
|
296
|
+
[id="plugins-{type}s-{plugin}-request_timeout"]
|
297
|
+
===== `request_timeout`
|
298
|
+
|
299
|
+
* Value type is <<number,number>>
|
300
|
+
* Default value is `60`
|
301
|
+
|
302
|
+
This module makes it easy to add a very fully configured HTTP client to logstash
|
303
|
+
based on [Manticore](https://github.com/cheald/manticore).
|
304
|
+
For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
|
305
|
+
Timeout (in seconds) for the entire request
|
306
|
+
|
307
|
+
[id="plugins-{type}s-{plugin}-retry_failed"]
|
308
|
+
===== `retry_failed`
|
309
|
+
|
310
|
+
* Value type is <<boolean,boolean>>
|
311
|
+
* Default value is `true`
|
312
|
+
|
313
|
+
Set this to false if you don't want this output to retry failed requests
|
314
|
+
|
315
|
+
[id="plugins-{type}s-{plugin}-retry_non_idempotent"]
|
316
|
+
===== `retry_non_idempotent`
|
317
|
+
|
318
|
+
* Value type is <<boolean,boolean>>
|
319
|
+
* Default value is `false`
|
320
|
+
|
321
|
+
If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
|
322
|
+
This only affects connectivity related errors (see related `automatic_retries` setting).
|
323
|
+
|
324
|
+
[id="plugins-{type}s-{plugin}-retryable_codes"]
|
325
|
+
===== `retryable_codes`
|
326
|
+
|
327
|
+
* Value type is <<number,number>>
|
328
|
+
* Default value is `[429, 500, 502, 503, 504]`
|
329
|
+
|
330
|
+
If encountered as response codes this plugin will retry these requests
|
331
|
+
|
332
|
+
[id="plugins-{type}s-{plugin}-socket_timeout"]
|
333
|
+
===== `socket_timeout`
|
334
|
+
|
335
|
+
* Value type is <<number,number>>
|
336
|
+
* Default value is `10`
|
337
|
+
|
338
|
+
Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
339
|
+
|
340
|
+
[id="plugins-{type}s-{plugin}-truststore"]
|
341
|
+
===== `truststore`
|
342
|
+
|
343
|
+
* Value type is <<path,path>>
|
344
|
+
* There is no default value for this setting.
|
345
|
+
|
346
|
+
If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
|
347
|
+
|
348
|
+
[id="plugins-{type}s-{plugin}-truststore_password"]
|
349
|
+
===== `truststore_password`
|
350
|
+
|
351
|
+
* Value type is <<password,password>>
|
352
|
+
* There is no default value for this setting.
|
353
|
+
|
354
|
+
Specify the truststore password here.
|
355
|
+
Note, most .jks files created with keytool require a password!
|
356
|
+
|
357
|
+
[id="plugins-{type}s-{plugin}-truststore_type"]
|
358
|
+
===== `truststore_type`
|
359
|
+
|
360
|
+
* Value type is <<string,string>>
|
361
|
+
* Default value is `"JKS"`
|
362
|
+
|
363
|
+
Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
364
|
+
|
365
|
+
[id="plugins-{type}s-{plugin}-url"]
|
366
|
+
===== `url`
|
367
|
+
|
368
|
+
* This is a required setting.
|
369
|
+
* Value type is <<string,string>>
|
370
|
+
* There is no default value for this setting.
|
371
|
+
|
372
|
+
URL to use
|
373
|
+
|
374
|
+
[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
|
375
|
+
===== `validate_after_inactivity`
|
376
|
+
|
377
|
+
* Value type is <<number,number>>
|
378
|
+
* Default value is `200`
|
379
|
+
|
380
|
+
How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
381
|
+
You may want to set this lower, possibly to 0 if you get connection errors regularly
|
382
|
+
Quoting the Apache commons docs (this client is based Apache Commmons):
|
383
|
+
'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
|
384
|
+
See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
385
|
+
|
386
|
+
|
387
|
+
|
388
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
389
|
+
include::{include_path}/{type}.asciidoc[]
|
390
|
+
|
391
|
+
:default_codec!:
|
@@ -0,0 +1,390 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require "logstash/json"
|
5
|
+
require "uri"
|
6
|
+
require "logstash/plugin_mixins/http_client"
|
7
|
+
require "zlib"
|
8
|
+
|
9
|
+
class LogStash::Outputs::HttpCodec < LogStash::Outputs::Base
|
10
|
+
include LogStash::PluginMixins::HttpClient
|
11
|
+
|
12
|
+
concurrency :shared
|
13
|
+
|
14
|
+
attr_accessor :is_batch
|
15
|
+
|
16
|
+
VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
|
17
|
+
|
18
|
+
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
19
|
+
::Manticore::Timeout,
|
20
|
+
::Manticore::SocketException,
|
21
|
+
::Manticore::ClientProtocolException,
|
22
|
+
::Manticore::ResolutionFailure,
|
23
|
+
::Manticore::SocketTimeout
|
24
|
+
]
|
25
|
+
|
26
|
+
# This output lets you send events to a
|
27
|
+
# generic HTTP(S) endpoint
|
28
|
+
#
|
29
|
+
# This output will execute up to 'pool_max' requests in parallel for performance.
|
30
|
+
# Consider this when tuning this plugin for performance.
|
31
|
+
#
|
32
|
+
# Additionally, note that when parallel execution is used strict ordering of events is not
|
33
|
+
# guaranteed!
|
34
|
+
#
|
35
|
+
# Beware, this gem does not yet support codecs. Please use the 'format' option for now.
|
36
|
+
|
37
|
+
config_name "httpcodec"
|
38
|
+
|
39
|
+
# URL to use
|
40
|
+
config :url, :validate => :string, :required => :true
|
41
|
+
|
42
|
+
# The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
|
43
|
+
config :http_method, :validate => VALID_METHODS, :required => :true
|
44
|
+
|
45
|
+
# Custom headers to use
|
46
|
+
# format is `headers => ["X-My-Header", "%{host}"]`
|
47
|
+
config :headers, :validate => :hash, :default => {}
|
48
|
+
|
49
|
+
# Content type
|
50
|
+
#
|
51
|
+
# If not specified, this defaults to the following:
|
52
|
+
#
|
53
|
+
# * if format is "json", "application/json"
|
54
|
+
# * if format is "form", "application/x-www-form-urlencoded"
|
55
|
+
config :content_type, :validate => :string
|
56
|
+
|
57
|
+
# Set this to false if you don't want this output to retry failed requests
|
58
|
+
config :retry_failed, :validate => :boolean, :default => true
|
59
|
+
|
60
|
+
# If encountered as response codes this plugin will retry these requests
|
61
|
+
config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
|
62
|
+
|
63
|
+
# If you would like to consider some non-2xx codes to be successes
|
64
|
+
# enumerate them here. Responses returning these codes will be considered successes
|
65
|
+
config :ignorable_codes, :validate => :number, :list => true
|
66
|
+
|
67
|
+
# This lets you choose the structure and parts of the event that are sent.
|
68
|
+
#
|
69
|
+
#
|
70
|
+
# For example:
|
71
|
+
# [source,ruby]
|
72
|
+
# mapping => {"foo" => "%{host}"
|
73
|
+
# "bar" => "%{type}"}
|
74
|
+
config :mapping, :validate => :hash
|
75
|
+
|
76
|
+
# Set the format of the http body.
|
77
|
+
#
|
78
|
+
# If form, then the body will be the mapping (or whole event) converted
|
79
|
+
# into a query parameter string, e.g. `foo=bar&baz=fizz...`
|
80
|
+
#
|
81
|
+
# If message, then the body will be the result of formatting the event according to message
|
82
|
+
#
|
83
|
+
# Otherwise, the event is sent as json.
|
84
|
+
config :format, :validate => ["json", "json_batch", "form", "message", "codec"], :default => "json"
|
85
|
+
|
86
|
+
# Set this to true if you want to enable gzip compression for your http requests
|
87
|
+
config :http_compression, :validate => :boolean, :default => false
|
88
|
+
|
89
|
+
config :message, :validate => :string
|
90
|
+
default :codec, "json"
|
91
|
+
|
92
|
+
def register
|
93
|
+
@codec.on_event do |event, payload|
|
94
|
+
payload
|
95
|
+
end
|
96
|
+
|
97
|
+
@http_method = @http_method.to_sym
|
98
|
+
|
99
|
+
# We count outstanding requests with this queue
|
100
|
+
# This queue tracks the requests to create backpressure
|
101
|
+
# When this queue is empty no new requests may be sent,
|
102
|
+
# tokens must be added back by the client on success
|
103
|
+
@request_tokens = SizedQueue.new(@pool_max)
|
104
|
+
@pool_max.times {|t| @request_tokens << true }
|
105
|
+
|
106
|
+
@requests = Array.new
|
107
|
+
|
108
|
+
if @content_type.nil?
|
109
|
+
case @format
|
110
|
+
when "form" ; @content_type = "application/x-www-form-urlencoded"
|
111
|
+
when "json" ; @content_type = "application/json"
|
112
|
+
when "json_batch" ; @content_type = "application/json"
|
113
|
+
when "message" ; @content_type = "text/plain"
|
114
|
+
when "codec" ; @content_type = "text/plain"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
@is_batch = @format == "json_batch"
|
119
|
+
|
120
|
+
@headers["Content-Type"] = @content_type
|
121
|
+
|
122
|
+
validate_format!
|
123
|
+
|
124
|
+
# Run named Timer as daemon thread
|
125
|
+
@timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
|
126
|
+
end # def register
|
127
|
+
|
128
|
+
def multi_receive(events)
|
129
|
+
return if events.empty?
|
130
|
+
send_events(events)
|
131
|
+
end
|
132
|
+
|
133
|
+
class RetryTimerTask < java.util.TimerTask
|
134
|
+
def initialize(pending, event, attempt)
|
135
|
+
@pending = pending
|
136
|
+
@event = event
|
137
|
+
@attempt = attempt
|
138
|
+
super()
|
139
|
+
end
|
140
|
+
|
141
|
+
def run
|
142
|
+
@pending << [@event, @attempt]
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def log_retryable_response(response)
|
147
|
+
if (response.code == 429)
|
148
|
+
@logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
|
149
|
+
else
|
150
|
+
@logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def log_error_response(response, url, event)
|
155
|
+
log_failure(
|
156
|
+
"Encountered non-2xx HTTP code #{response.code}",
|
157
|
+
:response_code => response.code,
|
158
|
+
:url => url,
|
159
|
+
:event => event
|
160
|
+
)
|
161
|
+
end
|
162
|
+
|
163
|
+
def send_events(events)
|
164
|
+
successes = java.util.concurrent.atomic.AtomicInteger.new(0)
|
165
|
+
failures = java.util.concurrent.atomic.AtomicInteger.new(0)
|
166
|
+
retries = java.util.concurrent.atomic.AtomicInteger.new(0)
|
167
|
+
event_count = @is_batch ? 1 : events.size
|
168
|
+
|
169
|
+
pending = Queue.new
|
170
|
+
if @is_batch
|
171
|
+
pending << [events, 0]
|
172
|
+
else
|
173
|
+
events.each {|e| pending << [e, 0]}
|
174
|
+
end
|
175
|
+
|
176
|
+
while popped = pending.pop
|
177
|
+
break if popped == :done
|
178
|
+
|
179
|
+
event, attempt = popped
|
180
|
+
|
181
|
+
action, event, attempt = send_event(event, attempt)
|
182
|
+
begin
|
183
|
+
action = :failure if action == :retry && !@retry_failed
|
184
|
+
|
185
|
+
case action
|
186
|
+
when :success
|
187
|
+
successes.incrementAndGet
|
188
|
+
when :retry
|
189
|
+
retries.incrementAndGet
|
190
|
+
|
191
|
+
next_attempt = attempt+1
|
192
|
+
sleep_for = sleep_for_attempt(next_attempt)
|
193
|
+
@logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
|
194
|
+
timer_task = RetryTimerTask.new(pending, event, next_attempt)
|
195
|
+
@timer.schedule(timer_task, sleep_for*1000)
|
196
|
+
when :failure
|
197
|
+
failures.incrementAndGet
|
198
|
+
else
|
199
|
+
raise "Unknown action #{action}"
|
200
|
+
end
|
201
|
+
|
202
|
+
if action == :success || action == :failure
|
203
|
+
if successes.get+failures.get == event_count
|
204
|
+
pending << :done
|
205
|
+
end
|
206
|
+
end
|
207
|
+
rescue => e
|
208
|
+
# This should never happen unless there's a flat out bug in the code
|
209
|
+
@logger.error("Error sending HTTP Request",
|
210
|
+
:class => e.class.name,
|
211
|
+
:message => e.message,
|
212
|
+
:backtrace => e.backtrace)
|
213
|
+
failures.incrementAndGet
|
214
|
+
raise e
|
215
|
+
end
|
216
|
+
end
|
217
|
+
rescue => e
|
218
|
+
@logger.error("Error in http output loop",
|
219
|
+
:class => e.class.name,
|
220
|
+
:message => e.message,
|
221
|
+
:backtrace => e.backtrace)
|
222
|
+
raise e
|
223
|
+
end
|
224
|
+
|
225
|
+
def sleep_for_attempt(attempt)
|
226
|
+
sleep_for = attempt**2
|
227
|
+
sleep_for = sleep_for <= 60 ? sleep_for : 60
|
228
|
+
(sleep_for/2) + (rand(0..sleep_for)/2)
|
229
|
+
end
|
230
|
+
|
231
|
+
def send_event(event, attempt)
|
232
|
+
body = event_body(event)
|
233
|
+
|
234
|
+
# Send the request
|
235
|
+
url = @is_batch ? @url : event.sprintf(@url)
|
236
|
+
headers = @is_batch ? @headers : event_headers(event)
|
237
|
+
|
238
|
+
# Compress the body and add appropriate header
|
239
|
+
if @http_compression == true
|
240
|
+
headers["Content-Encoding"] = "gzip"
|
241
|
+
body = gzip(body)
|
242
|
+
end
|
243
|
+
|
244
|
+
# Create an async request
|
245
|
+
response = client.send(@http_method, url, :body => body, :headers => headers).call
|
246
|
+
|
247
|
+
if !response_success?(response)
|
248
|
+
if retryable_response?(response)
|
249
|
+
log_retryable_response(response)
|
250
|
+
return :retry, event, attempt
|
251
|
+
else
|
252
|
+
log_error_response(response, url, event)
|
253
|
+
return :failure, event, attempt
|
254
|
+
end
|
255
|
+
else
|
256
|
+
return :success, event, attempt
|
257
|
+
end
|
258
|
+
|
259
|
+
rescue => exception
|
260
|
+
will_retry = retryable_exception?(exception)
|
261
|
+
log_failure("Could not fetch URL",
|
262
|
+
:url => url,
|
263
|
+
:method => @http_method,
|
264
|
+
:body => body,
|
265
|
+
:headers => headers,
|
266
|
+
:message => exception.message,
|
267
|
+
:class => exception.class.name,
|
268
|
+
:backtrace => exception.backtrace,
|
269
|
+
:will_retry => will_retry
|
270
|
+
)
|
271
|
+
|
272
|
+
if will_retry
|
273
|
+
return :retry, event, attempt
|
274
|
+
else
|
275
|
+
return :failure, event, attempt
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
def close
|
280
|
+
@timer.cancel
|
281
|
+
client.close
|
282
|
+
end
|
283
|
+
|
284
|
+
private
|
285
|
+
|
286
|
+
def response_success?(response)
|
287
|
+
code = response.code
|
288
|
+
return true if @ignorable_codes && @ignorable_codes.include?(code)
|
289
|
+
return code >= 200 && code <= 299
|
290
|
+
end
|
291
|
+
|
292
|
+
def retryable_response?(response)
|
293
|
+
@retryable_codes && @retryable_codes.include?(response.code)
|
294
|
+
end
|
295
|
+
|
296
|
+
def retryable_exception?(exception)
|
297
|
+
RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
|
298
|
+
end
|
299
|
+
|
300
|
+
# This is split into a separate method mostly to help testing
|
301
|
+
def log_failure(message, opts)
|
302
|
+
@logger.error("[HTTP Output Failure] #{message}", opts)
|
303
|
+
end
|
304
|
+
|
305
|
+
# Format the HTTP body
|
306
|
+
def event_body(event)
|
307
|
+
# TODO: Create an HTTP post data codec, use that here
|
308
|
+
if @format == "json"
|
309
|
+
LogStash::Json.dump(map_event(event))
|
310
|
+
elsif @format == "message"
|
311
|
+
event.sprintf(@message)
|
312
|
+
elsif @format == "codec"
|
313
|
+
@codec.encode(event)
|
314
|
+
elsif @format == "json_batch"
|
315
|
+
LogStash::Json.dump(event.map {|e| map_event(e) })
|
316
|
+
else
|
317
|
+
encode(map_event(event))
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
# gzip data
|
322
|
+
def gzip(data)
|
323
|
+
gz = StringIO.new
|
324
|
+
gz.set_encoding("BINARY")
|
325
|
+
z = Zlib::GzipWriter.new(gz)
|
326
|
+
z.write(data)
|
327
|
+
z.close
|
328
|
+
gz.string
|
329
|
+
end
|
330
|
+
|
331
|
+
def convert_mapping(mapping, event)
|
332
|
+
if mapping.is_a?(Hash)
|
333
|
+
mapping.reduce({}) do |acc, kv|
|
334
|
+
k, v = kv
|
335
|
+
acc[k] = convert_mapping(v, event)
|
336
|
+
acc
|
337
|
+
end
|
338
|
+
elsif mapping.is_a?(Array)
|
339
|
+
mapping.map { |elem| convert_mapping(elem, event) }
|
340
|
+
else
|
341
|
+
event.sprintf(mapping)
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
def map_event(event)
|
346
|
+
if @mapping
|
347
|
+
convert_mapping(@mapping, event)
|
348
|
+
else
|
349
|
+
event.to_hash
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
def event_headers(event)
|
354
|
+
custom_headers(event) || {}
|
355
|
+
end
|
356
|
+
|
357
|
+
def custom_headers(event)
|
358
|
+
return nil unless @headers
|
359
|
+
|
360
|
+
@headers.reduce({}) do |acc,kv|
|
361
|
+
k,v = kv
|
362
|
+
acc[k] = event.sprintf(v)
|
363
|
+
acc
|
364
|
+
end
|
365
|
+
end
|
366
|
+
|
367
|
+
#TODO Extract this to a codec
|
368
|
+
def encode(hash)
|
369
|
+
return hash.collect do |key, value|
|
370
|
+
CGI.escape(key) + "=" + CGI.escape(value.to_s)
|
371
|
+
end.join("&")
|
372
|
+
end
|
373
|
+
|
374
|
+
|
375
|
+
def validate_format!
|
376
|
+
if @format == "message"
|
377
|
+
if @message.nil?
|
378
|
+
raise "message must be set if message format is used"
|
379
|
+
end
|
380
|
+
|
381
|
+
if @content_type.nil?
|
382
|
+
raise "content_type must be set if message format is used"
|
383
|
+
end
|
384
|
+
|
385
|
+
unless @mapping.nil?
|
386
|
+
@logger.warn "mapping is not supported and will be ignored if message format is used"
|
387
|
+
end
|
388
|
+
end
|
389
|
+
end
|
390
|
+
end
|