logstash-output-httpcodec 5.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,391 @@
1
+ :plugin: http
2
+ :type: output
3
+ :default_codec: plain
4
+
5
+ ///////////////////////////////////////////
6
+ START - GENERATED VARIABLES, DO NOT EDIT!
7
+ ///////////////////////////////////////////
8
+ :version: %VERSION%
9
+ :release_date: %RELEASE_DATE%
10
+ :changelog_url: %CHANGELOG_URL%
11
+ :include_path: ../../../../logstash/docs/include
12
+ ///////////////////////////////////////////
13
+ END - GENERATED VARIABLES, DO NOT EDIT!
14
+ ///////////////////////////////////////////
15
+
16
+ [id="plugins-{type}s-{plugin}"]
17
+
18
+ === Http output plugin
19
+
20
+ include::{include_path}/plugin_header.asciidoc[]
21
+
22
+ ==== Description
23
+
24
+ This output lets you send events to a generic HTTP(S) endpoint.
25
+
26
+ This output will execute up to 'pool_max' requests in parallel for performance.
27
+ Consider this when tuning this plugin for performance.
28
+
29
+ Additionally, note that when parallel execution is used strict ordering of events is not
30
+ guaranteed!
31
+
32
+ Beware, to use codecs, please use the 'format' option for now.
33
+
34
+ [id="plugins-{type}s-{plugin}-options"]
35
+ ==== Http Output Configuration Options
36
+
37
+ This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
38
+
39
+ [cols="<,<,<",options="header",]
40
+ |=======================================================================
41
+ |Setting |Input type|Required
42
+ | <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
43
+ | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
44
+ | <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
45
+ | <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
46
+ | <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
47
+ | <<plugins-{type}s-{plugin}-content_type>> |<<string,string>>|No
48
+ | <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
49
+ | <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
50
+ | <<plugins-{type}s-{plugin}-format>> |<<string,string>>, one of `["json", "json_batch", "form", "message"]`|No
51
+ | <<plugins-{type}s-{plugin}-headers>> |<<hash,hash>>|No
52
+ | <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
53
+ | <<plugins-{type}s-{plugin}-http_method>> |<<string,string>>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes
54
+ | <<plugins-{type}s-{plugin}-ignorable_codes>> |<<number,number>>|No
55
+ | <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
56
+ | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
57
+ | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
58
+ | <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
59
+ | <<plugins-{type}s-{plugin}-mapping>> |<<hash,hash>>|No
60
+ | <<plugins-{type}s-{plugin}-message>> |<<string,string>>|No
61
+ | <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
62
+ | <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
63
+ | <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
64
+ | <<plugins-{type}s-{plugin}-request_timeout>> |<<number,number>>|No
65
+ | <<plugins-{type}s-{plugin}-retry_failed>> |<<boolean,boolean>>|No
66
+ | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
67
+ | <<plugins-{type}s-{plugin}-retryable_codes>> |<<number,number>>|No
68
+ | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
69
+ | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
70
+ | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
71
+ | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
72
+ | <<plugins-{type}s-{plugin}-url>> |<<string,string>>|Yes
73
+ | <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
74
+ |=======================================================================
75
+
76
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
77
+ output plugins.
78
+
79
+ &nbsp;
80
+
81
+ [id="plugins-{type}s-{plugin}-automatic_retries"]
82
+ ===== `automatic_retries`
83
+
84
+ * Value type is <<number,number>>
85
+ * Default value is `1`
86
+
87
+ How many times should the client retry a failing URL. We highly recommend NOT setting this value
88
+ to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
89
+ Only IO related failures will be retried, such as connection timeouts and unreachable hosts.
90
+ Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting,
91
+ unless `retry_failed` is set.
92
+ Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
93
+
94
+ [id="plugins-{type}s-{plugin}-cacert"]
95
+ ===== `cacert`
96
+
97
+ * Value type is <<path,path>>
98
+ * There is no default value for this setting.
99
+
100
+ If you need to use a custom X.509 CA (.pem certs) specify the path to that here
101
+
102
+ [id="plugins-{type}s-{plugin}-client_cert"]
103
+ ===== `client_cert`
104
+
105
+ * Value type is <<path,path>>
106
+ * There is no default value for this setting.
107
+
108
+ If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
109
+
110
+ [id="plugins-{type}s-{plugin}-client_key"]
111
+ ===== `client_key`
112
+
113
+ * Value type is <<path,path>>
114
+ * There is no default value for this setting.
115
+
116
+ If you're using a client certificate specify the path to the encryption key here
117
+
118
+ [id="plugins-{type}s-{plugin}-connect_timeout"]
119
+ ===== `connect_timeout`
120
+
121
+ * Value type is <<number,number>>
122
+ * Default value is `10`
123
+
124
+ Timeout (in seconds) to wait for a connection to be established. Default is `10s`
125
+
126
+ [id="plugins-{type}s-{plugin}-content_type"]
127
+ ===== `content_type`
128
+
129
+ * Value type is <<string,string>>
130
+ * There is no default value for this setting.
131
+
132
+ Content type
133
+
134
+ If not specified, this defaults to the following:
135
+
136
+ * if format is "json", "application/json"
137
+ * if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request.
138
+ * if format is "form", "application/x-www-form-urlencoded"
139
+
140
+ [id="plugins-{type}s-{plugin}-cookies"]
141
+ ===== `cookies`
142
+
143
+ * Value type is <<boolean,boolean>>
144
+ * Default value is `true`
145
+
146
+ Enable cookie support. With this enabled the client will persist cookies
147
+ across requests as a normal web browser would. Enabled by default
148
+
149
+ [id="plugins-{type}s-{plugin}-follow_redirects"]
150
+ ===== `follow_redirects`
151
+
152
+ * Value type is <<boolean,boolean>>
153
+ * Default value is `true`
154
+
155
+ Should redirects be followed? Defaults to `true`
156
+
157
+ [id="plugins-{type}s-{plugin}-format"]
158
+ ===== `format`
159
+
160
+ * Value can be any of: `json`, `json_batch`, `form`, `message`, `codec`
161
+ * Default value is `"json"`
162
+
163
+ Set the format of the http body.
164
+
165
+ If json_batch, each batch of events received by this output will be placed
166
+ into a single JSON array and sent in one request. This is particularly useful
167
+ for high throughput scenarios such as sending data between Logstash instaces.
168
+
169
+ If form, then the body will be the mapping (or whole event) converted
170
+ into a query parameter string, e.g. `foo=bar&baz=fizz...`
171
+
172
+ If message, then the body will be the result of formatting the event according to message
173
+
174
+ Otherwise, the event is sent as json.
175
+
176
+ [id="plugins-{type}s-{plugin}-headers"]
177
+ ===== `headers`
178
+
179
+ * Value type is <<hash,hash>>
180
+ * There is no default value for this setting.
181
+
182
+ Custom headers to use
183
+ format is `headers => ["X-My-Header", "%{host}"]`
184
+
185
+ [id="plugins-{type}s-{plugin}-http_compression"]
186
+ ===== `http_compression`
187
+
188
+ * Value type is <<boolean,boolean>>
189
+ * Default value is `false`
190
+
191
+ Enable request compression support. With this enabled the plugin will compress
192
+ http requests using gzip.
193
+
194
+ [id="plugins-{type}s-{plugin}-http_method"]
195
+ ===== `http_method`
196
+
197
+ * This is a required setting.
198
+ * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head`
199
+ * There is no default value for this setting.
200
+
201
+ The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
202
+
203
+ [id="plugins-{type}s-{plugin}-ignorable_codes"]
204
+ ===== `ignorable_codes`
205
+
206
+ * Value type is <<number,number>>
207
+ * There is no default value for this setting.
208
+
209
+ If you would like to consider some non-2xx codes to be successes
210
+ enumerate them here. Responses returning these codes will be considered successes
211
+
212
+ [id="plugins-{type}s-{plugin}-keepalive"]
213
+ ===== `keepalive`
214
+
215
+ * Value type is <<boolean,boolean>>
216
+ * Default value is `true`
217
+
218
+ Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
219
+ one with this to fix interactions with broken keepalive implementations.
220
+
221
+ [id="plugins-{type}s-{plugin}-keystore"]
222
+ ===== `keystore`
223
+
224
+ * Value type is <<path,path>>
225
+ * There is no default value for this setting.
226
+
227
+ If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
228
+
229
+ [id="plugins-{type}s-{plugin}-keystore_password"]
230
+ ===== `keystore_password`
231
+
232
+ * Value type is <<password,password>>
233
+ * There is no default value for this setting.
234
+
235
+ Specify the keystore password here.
236
+ Note, most .jks files created with keytool require a password!
237
+
238
+ [id="plugins-{type}s-{plugin}-keystore_type"]
239
+ ===== `keystore_type`
240
+
241
+ * Value type is <<string,string>>
242
+ * Default value is `"JKS"`
243
+
244
+ Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
245
+
246
+ [id="plugins-{type}s-{plugin}-mapping"]
247
+ ===== `mapping`
248
+
249
+ * Value type is <<hash,hash>>
250
+ * There is no default value for this setting.
251
+
252
+ This lets you choose the structure and parts of the event that are sent.
253
+
254
+
255
+ For example:
256
+ [source,ruby]
257
+ mapping => {"foo" => "%{host}"
258
+ "bar" => "%{type}"}
259
+
260
+ [id="plugins-{type}s-{plugin}-message"]
261
+ ===== `message`
262
+
263
+ * Value type is <<string,string>>
264
+ * There is no default value for this setting.
265
+
266
+
267
+
268
+ [id="plugins-{type}s-{plugin}-pool_max"]
269
+ ===== `pool_max`
270
+
271
+ * Value type is <<number,number>>
272
+ * Default value is `50`
273
+
274
+ Max number of concurrent connections. Defaults to `50`
275
+
276
+ [id="plugins-{type}s-{plugin}-pool_max_per_route"]
277
+ ===== `pool_max_per_route`
278
+
279
+ * Value type is <<number,number>>
280
+ * Default value is `25`
281
+
282
+ Max number of concurrent connections to a single host. Defaults to `25`
283
+
284
+ [id="plugins-{type}s-{plugin}-proxy"]
285
+ ===== `proxy`
286
+
287
+ * Value type is <<string,string>>
288
+ * There is no default value for this setting.
289
+
290
+ If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
291
+
292
+ 1. Proxy host in form: `http://proxy.org:1234`
293
+ 2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
294
+ 3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}`
295
+
296
+ [id="plugins-{type}s-{plugin}-request_timeout"]
297
+ ===== `request_timeout`
298
+
299
+ * Value type is <<number,number>>
300
+ * Default value is `60`
301
+
302
+ This module makes it easy to add a very fully configured HTTP client to logstash
303
+ based on [Manticore](https://github.com/cheald/manticore).
304
+ For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
305
+ Timeout (in seconds) for the entire request
306
+
307
+ [id="plugins-{type}s-{plugin}-retry_failed"]
308
+ ===== `retry_failed`
309
+
310
+ * Value type is <<boolean,boolean>>
311
+ * Default value is `true`
312
+
313
+ Set this to false if you don't want this output to retry failed requests
314
+
315
+ [id="plugins-{type}s-{plugin}-retry_non_idempotent"]
316
+ ===== `retry_non_idempotent`
317
+
318
+ * Value type is <<boolean,boolean>>
319
+ * Default value is `false`
320
+
321
+ If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
322
+ This only affects connectivity related errors (see related `automatic_retries` setting).
323
+
324
+ [id="plugins-{type}s-{plugin}-retryable_codes"]
325
+ ===== `retryable_codes`
326
+
327
+ * Value type is <<number,number>>
328
+ * Default value is `[429, 500, 502, 503, 504]`
329
+
330
+ If encountered as response codes this plugin will retry these requests
331
+
332
+ [id="plugins-{type}s-{plugin}-socket_timeout"]
333
+ ===== `socket_timeout`
334
+
335
+ * Value type is <<number,number>>
336
+ * Default value is `10`
337
+
338
+ Timeout (in seconds) to wait for data on the socket. Default is `10s`
339
+
340
+ [id="plugins-{type}s-{plugin}-truststore"]
341
+ ===== `truststore`
342
+
343
+ * Value type is <<path,path>>
344
+ * There is no default value for this setting.
345
+
346
+ If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
347
+
348
+ [id="plugins-{type}s-{plugin}-truststore_password"]
349
+ ===== `truststore_password`
350
+
351
+ * Value type is <<password,password>>
352
+ * There is no default value for this setting.
353
+
354
+ Specify the truststore password here.
355
+ Note, most .jks files created with keytool require a password!
356
+
357
+ [id="plugins-{type}s-{plugin}-truststore_type"]
358
+ ===== `truststore_type`
359
+
360
+ * Value type is <<string,string>>
361
+ * Default value is `"JKS"`
362
+
363
+ Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
364
+
365
+ [id="plugins-{type}s-{plugin}-url"]
366
+ ===== `url`
367
+
368
+ * This is a required setting.
369
+ * Value type is <<string,string>>
370
+ * There is no default value for this setting.
371
+
372
+ URL to use
373
+
374
+ [id="plugins-{type}s-{plugin}-validate_after_inactivity"]
375
+ ===== `validate_after_inactivity`
376
+
377
+ * Value type is <<number,number>>
378
+ * Default value is `200`
379
+
380
+ How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
381
+ You may want to set this lower, possibly to 0 if you get connection errors regularly
382
+ Quoting the Apache commons docs (this client is based Apache Commmons):
383
+ 'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
384
+ See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
385
+
386
+
387
+
388
+ [id="plugins-{type}s-{plugin}-common-options"]
389
+ include::{include_path}/{type}.asciidoc[]
390
+
391
+ :default_codec!:
@@ -0,0 +1,390 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "logstash/namespace"
4
+ require "logstash/json"
5
+ require "uri"
6
+ require "logstash/plugin_mixins/http_client"
7
+ require "zlib"
8
+
9
+ class LogStash::Outputs::HttpCodec < LogStash::Outputs::Base
10
+ include LogStash::PluginMixins::HttpClient
11
+
12
+ concurrency :shared
13
+
14
+ attr_accessor :is_batch
15
+
16
+ VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
17
+
18
+ RETRYABLE_MANTICORE_EXCEPTIONS = [
19
+ ::Manticore::Timeout,
20
+ ::Manticore::SocketException,
21
+ ::Manticore::ClientProtocolException,
22
+ ::Manticore::ResolutionFailure,
23
+ ::Manticore::SocketTimeout
24
+ ]
25
+
26
+ # This output lets you send events to a
27
+ # generic HTTP(S) endpoint
28
+ #
29
+ # This output will execute up to 'pool_max' requests in parallel for performance.
30
+ # Consider this when tuning this plugin for performance.
31
+ #
32
+ # Additionally, note that when parallel execution is used strict ordering of events is not
33
+ # guaranteed!
34
+ #
35
+ # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
36
+
37
+ config_name "httpcodec"
38
+
39
+ # URL to use
40
+ config :url, :validate => :string, :required => :true
41
+
42
+ # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
43
+ config :http_method, :validate => VALID_METHODS, :required => :true
44
+
45
+ # Custom headers to use
46
+ # format is `headers => ["X-My-Header", "%{host}"]`
47
+ config :headers, :validate => :hash, :default => {}
48
+
49
+ # Content type
50
+ #
51
+ # If not specified, this defaults to the following:
52
+ #
53
+ # * if format is "json", "application/json"
54
+ # * if format is "form", "application/x-www-form-urlencoded"
55
+ config :content_type, :validate => :string
56
+
57
+ # Set this to false if you don't want this output to retry failed requests
58
+ config :retry_failed, :validate => :boolean, :default => true
59
+
60
+ # If encountered as response codes this plugin will retry these requests
61
+ config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
62
+
63
+ # If you would like to consider some non-2xx codes to be successes
64
+ # enumerate them here. Responses returning these codes will be considered successes
65
+ config :ignorable_codes, :validate => :number, :list => true
66
+
67
+ # This lets you choose the structure and parts of the event that are sent.
68
+ #
69
+ #
70
+ # For example:
71
+ # [source,ruby]
72
+ # mapping => {"foo" => "%{host}"
73
+ # "bar" => "%{type}"}
74
+ config :mapping, :validate => :hash
75
+
76
+ # Set the format of the http body.
77
+ #
78
+ # If form, then the body will be the mapping (or whole event) converted
79
+ # into a query parameter string, e.g. `foo=bar&baz=fizz...`
80
+ #
81
+ # If message, then the body will be the result of formatting the event according to message
82
+ #
83
+ # Otherwise, the event is sent as json.
84
+ config :format, :validate => ["json", "json_batch", "form", "message", "codec"], :default => "json"
85
+
86
+ # Set this to true if you want to enable gzip compression for your http requests
87
+ config :http_compression, :validate => :boolean, :default => false
88
+
89
+ config :message, :validate => :string
90
+ default :codec, "json"
91
+
92
+ def register
93
+ @codec.on_event do |event, payload|
94
+ payload
95
+ end
96
+
97
+ @http_method = @http_method.to_sym
98
+
99
+ # We count outstanding requests with this queue
100
+ # This queue tracks the requests to create backpressure
101
+ # When this queue is empty no new requests may be sent,
102
+ # tokens must be added back by the client on success
103
+ @request_tokens = SizedQueue.new(@pool_max)
104
+ @pool_max.times {|t| @request_tokens << true }
105
+
106
+ @requests = Array.new
107
+
108
+ if @content_type.nil?
109
+ case @format
110
+ when "form" ; @content_type = "application/x-www-form-urlencoded"
111
+ when "json" ; @content_type = "application/json"
112
+ when "json_batch" ; @content_type = "application/json"
113
+ when "message" ; @content_type = "text/plain"
114
+ when "codec" ; @content_type = "text/plain"
115
+ end
116
+ end
117
+
118
+ @is_batch = @format == "json_batch"
119
+
120
+ @headers["Content-Type"] = @content_type
121
+
122
+ validate_format!
123
+
124
+ # Run named Timer as daemon thread
125
+ @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
126
+ end # def register
127
+
128
+ def multi_receive(events)
129
+ return if events.empty?
130
+ send_events(events)
131
+ end
132
+
133
+ class RetryTimerTask < java.util.TimerTask
134
+ def initialize(pending, event, attempt)
135
+ @pending = pending
136
+ @event = event
137
+ @attempt = attempt
138
+ super()
139
+ end
140
+
141
+ def run
142
+ @pending << [@event, @attempt]
143
+ end
144
+ end
145
+
146
+ def log_retryable_response(response)
147
+ if (response.code == 429)
148
+ @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
149
+ else
150
+ @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
151
+ end
152
+ end
153
+
154
+ def log_error_response(response, url, event)
155
+ log_failure(
156
+ "Encountered non-2xx HTTP code #{response.code}",
157
+ :response_code => response.code,
158
+ :url => url,
159
+ :event => event
160
+ )
161
+ end
162
+
163
+ def send_events(events)
164
+ successes = java.util.concurrent.atomic.AtomicInteger.new(0)
165
+ failures = java.util.concurrent.atomic.AtomicInteger.new(0)
166
+ retries = java.util.concurrent.atomic.AtomicInteger.new(0)
167
+ event_count = @is_batch ? 1 : events.size
168
+
169
+ pending = Queue.new
170
+ if @is_batch
171
+ pending << [events, 0]
172
+ else
173
+ events.each {|e| pending << [e, 0]}
174
+ end
175
+
176
+ while popped = pending.pop
177
+ break if popped == :done
178
+
179
+ event, attempt = popped
180
+
181
+ action, event, attempt = send_event(event, attempt)
182
+ begin
183
+ action = :failure if action == :retry && !@retry_failed
184
+
185
+ case action
186
+ when :success
187
+ successes.incrementAndGet
188
+ when :retry
189
+ retries.incrementAndGet
190
+
191
+ next_attempt = attempt+1
192
+ sleep_for = sleep_for_attempt(next_attempt)
193
+ @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
194
+ timer_task = RetryTimerTask.new(pending, event, next_attempt)
195
+ @timer.schedule(timer_task, sleep_for*1000)
196
+ when :failure
197
+ failures.incrementAndGet
198
+ else
199
+ raise "Unknown action #{action}"
200
+ end
201
+
202
+ if action == :success || action == :failure
203
+ if successes.get+failures.get == event_count
204
+ pending << :done
205
+ end
206
+ end
207
+ rescue => e
208
+ # This should never happen unless there's a flat out bug in the code
209
+ @logger.error("Error sending HTTP Request",
210
+ :class => e.class.name,
211
+ :message => e.message,
212
+ :backtrace => e.backtrace)
213
+ failures.incrementAndGet
214
+ raise e
215
+ end
216
+ end
217
+ rescue => e
218
+ @logger.error("Error in http output loop",
219
+ :class => e.class.name,
220
+ :message => e.message,
221
+ :backtrace => e.backtrace)
222
+ raise e
223
+ end
224
+
225
+ def sleep_for_attempt(attempt)
226
+ sleep_for = attempt**2
227
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
228
+ (sleep_for/2) + (rand(0..sleep_for)/2)
229
+ end
230
+
231
+ def send_event(event, attempt)
232
+ body = event_body(event)
233
+
234
+ # Send the request
235
+ url = @is_batch ? @url : event.sprintf(@url)
236
+ headers = @is_batch ? @headers : event_headers(event)
237
+
238
+ # Compress the body and add appropriate header
239
+ if @http_compression == true
240
+ headers["Content-Encoding"] = "gzip"
241
+ body = gzip(body)
242
+ end
243
+
244
+ # Create an async request
245
+ response = client.send(@http_method, url, :body => body, :headers => headers).call
246
+
247
+ if !response_success?(response)
248
+ if retryable_response?(response)
249
+ log_retryable_response(response)
250
+ return :retry, event, attempt
251
+ else
252
+ log_error_response(response, url, event)
253
+ return :failure, event, attempt
254
+ end
255
+ else
256
+ return :success, event, attempt
257
+ end
258
+
259
+ rescue => exception
260
+ will_retry = retryable_exception?(exception)
261
+ log_failure("Could not fetch URL",
262
+ :url => url,
263
+ :method => @http_method,
264
+ :body => body,
265
+ :headers => headers,
266
+ :message => exception.message,
267
+ :class => exception.class.name,
268
+ :backtrace => exception.backtrace,
269
+ :will_retry => will_retry
270
+ )
271
+
272
+ if will_retry
273
+ return :retry, event, attempt
274
+ else
275
+ return :failure, event, attempt
276
+ end
277
+ end
278
+
279
+ def close
280
+ @timer.cancel
281
+ client.close
282
+ end
283
+
284
+ private
285
+
286
+ def response_success?(response)
287
+ code = response.code
288
+ return true if @ignorable_codes && @ignorable_codes.include?(code)
289
+ return code >= 200 && code <= 299
290
+ end
291
+
292
+ def retryable_response?(response)
293
+ @retryable_codes && @retryable_codes.include?(response.code)
294
+ end
295
+
296
+ def retryable_exception?(exception)
297
+ RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
298
+ end
299
+
300
+ # This is split into a separate method mostly to help testing
301
+ def log_failure(message, opts)
302
+ @logger.error("[HTTP Output Failure] #{message}", opts)
303
+ end
304
+
305
+ # Format the HTTP body
306
+ def event_body(event)
307
+ # TODO: Create an HTTP post data codec, use that here
308
+ if @format == "json"
309
+ LogStash::Json.dump(map_event(event))
310
+ elsif @format == "message"
311
+ event.sprintf(@message)
312
+ elsif @format == "codec"
313
+ @codec.encode(event)
314
+ elsif @format == "json_batch"
315
+ LogStash::Json.dump(event.map {|e| map_event(e) })
316
+ else
317
+ encode(map_event(event))
318
+ end
319
+ end
320
+
321
+ # gzip data
322
+ def gzip(data)
323
+ gz = StringIO.new
324
+ gz.set_encoding("BINARY")
325
+ z = Zlib::GzipWriter.new(gz)
326
+ z.write(data)
327
+ z.close
328
+ gz.string
329
+ end
330
+
331
+ def convert_mapping(mapping, event)
332
+ if mapping.is_a?(Hash)
333
+ mapping.reduce({}) do |acc, kv|
334
+ k, v = kv
335
+ acc[k] = convert_mapping(v, event)
336
+ acc
337
+ end
338
+ elsif mapping.is_a?(Array)
339
+ mapping.map { |elem| convert_mapping(elem, event) }
340
+ else
341
+ event.sprintf(mapping)
342
+ end
343
+ end
344
+
345
+ def map_event(event)
346
+ if @mapping
347
+ convert_mapping(@mapping, event)
348
+ else
349
+ event.to_hash
350
+ end
351
+ end
352
+
353
+ def event_headers(event)
354
+ custom_headers(event) || {}
355
+ end
356
+
357
+ def custom_headers(event)
358
+ return nil unless @headers
359
+
360
+ @headers.reduce({}) do |acc,kv|
361
+ k,v = kv
362
+ acc[k] = event.sprintf(v)
363
+ acc
364
+ end
365
+ end
366
+
367
+ #TODO Extract this to a codec
368
+ def encode(hash)
369
+ return hash.collect do |key, value|
370
+ CGI.escape(key) + "=" + CGI.escape(value.to_s)
371
+ end.join("&")
372
+ end
373
+
374
+
375
+ def validate_format!
376
+ if @format == "message"
377
+ if @message.nil?
378
+ raise "message must be set if message format is used"
379
+ end
380
+
381
+ if @content_type.nil?
382
+ raise "content_type must be set if message format is used"
383
+ end
384
+
385
+ unless @mapping.nil?
386
+ @logger.warn "mapping is not supported and will be ignored if message format is used"
387
+ end
388
+ end
389
+ end
390
+ end