logstash-promtail-http-input 4.0.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +112 -0
- data/DEVELOPER.md +1 -0
- data/Gemfile +11 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +137 -0
- data/VERSION +1 -0
- data/docs/index.asciidoc +346 -0
- data/lib/logstash-input-http_jars.rb +10 -0
- data/lib/logstash/inputs/http.rb +316 -0
- data/lib/logstash/inputs/http/message_handler.rb +49 -0
- data/lib/logstash/inputs/http/tls.rb +40 -0
- data/lib/logstash/util/http_compressed_requests.rb +39 -0
- data/lib/tasks/build.rake +16 -0
- data/logstash-promtail-http-input.gemspec +33 -0
- data/spec/inputs/http_spec.rb +522 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.12.3/jackson-core-2.12.3.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.12.3/jackson-databind-2.12.3.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.14.0/protobuf-java-3.14.0.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/plugins/input/http/logstash-input-http/3.3.7/logstash-input-http-3.3.7.jar +0 -0
- data/vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar +0 -0
- metadata +181 -0
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
4.0.1
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,346 @@
|
|
1
|
+
:plugin: http
|
2
|
+
:type: input
|
3
|
+
:default_codec: plain
|
4
|
+
|
5
|
+
///////////////////////////////////////////
|
6
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
7
|
+
///////////////////////////////////////////
|
8
|
+
:version: %VERSION%
|
9
|
+
:release_date: %RELEASE_DATE%
|
10
|
+
:changelog_url: %CHANGELOG_URL%
|
11
|
+
:include_path: ../../../../logstash/docs/include
|
12
|
+
///////////////////////////////////////////
|
13
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
14
|
+
///////////////////////////////////////////
|
15
|
+
|
16
|
+
[id="plugins-{type}s-{plugin}"]
|
17
|
+
|
18
|
+
=== Http input plugin
|
19
|
+
|
20
|
+
include::{include_path}/plugin_header.asciidoc[]
|
21
|
+
|
22
|
+
==== Description
|
23
|
+
|
24
|
+
Using this input you can receive single or multiline events over http(s).
|
25
|
+
Applications can send an HTTP request to the endpoint started by this input and
|
26
|
+
Logstash will convert it into an event for subsequent processing. Users
|
27
|
+
can pass plain text, JSON, or any formatted data and use a corresponding codec with this
|
28
|
+
input. For Content-Type `application/json` the `json` codec is used, but for all other
|
29
|
+
data formats, `plain` codec is used.
|
30
|
+
|
31
|
+
This input can also be used to receive webhook requests to integrate with other services
|
32
|
+
and applications. By taking advantage of the vast plugin ecosystem available in Logstash
|
33
|
+
you can trigger actionable events right from your application.
|
34
|
+
|
35
|
+
==== Blocking Behavior
|
36
|
+
|
37
|
+
The HTTP protocol doesn't deal well with long running requests. This plugin will either return
|
38
|
+
a 429 (busy) error when Logstash is backlogged, or it will time out the request.
|
39
|
+
|
40
|
+
If a 429 error is encountered clients should sleep, backing off exponentially with some random
|
41
|
+
jitter, then retry their request.
|
42
|
+
|
43
|
+
This plugin will block if the Logstash queue is blocked and there are available HTTP input threads.
|
44
|
+
This will cause most HTTP clients to time out. Sent events will still be processed in this case. This
|
45
|
+
behavior is not optimal and will be changed in a future release. In the future, this plugin will always
|
46
|
+
return a 429 if the queue is busy, and will not time out in the event of a busy queue.
|
47
|
+
|
48
|
+
==== Security
|
49
|
+
This plugin supports standard HTTP basic authentication headers to identify the requester.
|
50
|
+
You can pass in a username, password combination while sending data to this input
|
51
|
+
|
52
|
+
You can also setup SSL and send data securely over https, with multiple options such as
|
53
|
+
validating the client's certificate.
|
54
|
+
|
55
|
+
[id="plugins-{type}s-{plugin}-codec-settings"]
|
56
|
+
==== Codec settings
|
57
|
+
This plugin has two configuration options for codecs: `codec` and `additional_codecs`.
|
58
|
+
|
59
|
+
Values in `additional_codecs` are prioritized over those specified in the
|
60
|
+
`codec` option. That is, the default `codec` is applied only if no codec
|
61
|
+
for the request's content-type is found in the `additional_codecs` setting.
|
62
|
+
|
63
|
+
[id="plugins-{type}s-{plugin}-options"]
|
64
|
+
==== Http Input Configuration Options
|
65
|
+
|
66
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
67
|
+
|
68
|
+
[cols="<,<,<",options="header",]
|
69
|
+
|=======================================================================
|
70
|
+
|Setting |Input type|Required
|
71
|
+
| <<plugins-{type}s-{plugin}-additional_codecs>> |<<hash,hash>>|No
|
72
|
+
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
73
|
+
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
74
|
+
| <<plugins-{type}s-{plugin}-keystore>> |<<path,path>>|No
|
75
|
+
| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
|
76
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
77
|
+
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
|
78
|
+
| <<plugins-{type}s-{plugin}-max_pending_requests>> |<<number,number>>|No
|
79
|
+
| <<plugins-{type}s-{plugin}-response_headers>> |<<hash,hash>>|No
|
80
|
+
| <<plugins-{type}s-{plugin}-response_code>> |<<number,number>>, one of `[200, 201, 202, 204]`|No
|
81
|
+
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
|
82
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No
|
83
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |<<array,array>>|No
|
84
|
+
| <<plugins-{type}s-{plugin}-ssl_handshake_timeout>> |<<number,number>>|No
|
85
|
+
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
86
|
+
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
87
|
+
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
88
|
+
| <<plugins-{type}s-{plugin}-threads>> |<<number,number>>|No
|
89
|
+
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|No
|
90
|
+
| <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|No
|
91
|
+
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
|
92
|
+
| <<plugins-{type}s-{plugin}-verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
93
|
+
|=======================================================================
|
94
|
+
|
95
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
96
|
+
input plugins.
|
97
|
+
|
98
|
+
|
99
|
+
|
100
|
+
[id="plugins-{type}s-{plugin}-additional_codecs"]
|
101
|
+
===== `additional_codecs`
|
102
|
+
|
103
|
+
* Value type is <<hash,hash>>
|
104
|
+
* Default value is `{"application/json"=>"json"}`
|
105
|
+
|
106
|
+
Apply specific codecs for specific content types.
|
107
|
+
The default codec will be applied only after this list is checked
|
108
|
+
and no codec for the request's content-type is found
|
109
|
+
|
110
|
+
[id="plugins-{type}s-{plugin}-cipher_suites"]
|
111
|
+
===== `cipher_suites`
|
112
|
+
|
113
|
+
* Value type is <<array,array>>
|
114
|
+
* Default value is `java.lang.String[TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
|
115
|
+
|
116
|
+
The list of ciphers suite to use, listed by priorities.
|
117
|
+
|
118
|
+
[id="plugins-{type}s-{plugin}-host"]
|
119
|
+
===== `host`
|
120
|
+
|
121
|
+
* Value type is <<string,string>>
|
122
|
+
* Default value is `"0.0.0.0"`
|
123
|
+
|
124
|
+
The host or ip to bind
|
125
|
+
|
126
|
+
[id="plugins-{type}s-{plugin}-keystore"]
|
127
|
+
===== `keystore`
|
128
|
+
|
129
|
+
* Value type is <<path,path>>
|
130
|
+
* There is no default value for this setting.
|
131
|
+
* This option is deprecated
|
132
|
+
|
133
|
+
The JKS keystore to validate the client's certificates
|
134
|
+
|
135
|
+
Note: This option is deprecated and it will be removed in the next major version of Logstash.
|
136
|
+
Use `ssl_certificate` and `ssl_key` instead.
|
137
|
+
|
138
|
+
[id="plugins-{type}s-{plugin}-keystore_password"]
|
139
|
+
===== `keystore_password`
|
140
|
+
|
141
|
+
* Value type is <<password,password>>
|
142
|
+
* There is no default value for this setting.
|
143
|
+
* This option is deprecated
|
144
|
+
|
145
|
+
Set the truststore password
|
146
|
+
|
147
|
+
Note: This option is deprecated and it will be removed in the next major version of Logstash.
|
148
|
+
Use `ssl_certificate` and `ssl_key` instead.
|
149
|
+
|
150
|
+
[id="plugins-{type}s-{plugin}-password"]
|
151
|
+
===== `password`
|
152
|
+
|
153
|
+
* Value type is <<password,password>>
|
154
|
+
* There is no default value for this setting.
|
155
|
+
|
156
|
+
Password for basic authorization
|
157
|
+
|
158
|
+
[id="plugins-{type}s-{plugin}-port"]
|
159
|
+
===== `port`
|
160
|
+
|
161
|
+
* Value type is <<number,number>>
|
162
|
+
* Default value is `8080`
|
163
|
+
|
164
|
+
The TCP port to bind to
|
165
|
+
|
166
|
+
[id="plugins-{type}s-{plugin}-max_content_length"]
|
167
|
+
===== `max_content_length`
|
168
|
+
|
169
|
+
* Value type is <<number,number>>
|
170
|
+
* Default value is 104857600
|
171
|
+
|
172
|
+
The max content of an HTTP request in bytes. It defaults to 100mb.
|
173
|
+
|
174
|
+
[id="plugins-{type}s-{plugin}-max_pending_requests"]
|
175
|
+
===== `max_pending_requests`
|
176
|
+
|
177
|
+
* Value type is <<number,number>>
|
178
|
+
* Default value is 200
|
179
|
+
|
180
|
+
Maximum number of incoming requests to store in a temporary queue before being processed by worker threads.
|
181
|
+
If a request arrives and the queue is full a 429 response will be returned immediately.
|
182
|
+
This queue exists to deal with micro bursts of events and to improve overall throughput,
|
183
|
+
so it should be changed very carefully as it can lead to memory pressure and impact performance.
|
184
|
+
If you need to deal both periodic or unforeseen spikes in incoming requests consider enabling the
|
185
|
+
Persistent Queue for the logstash pipeline.
|
186
|
+
|
187
|
+
[id="plugins-{type}s-{plugin}-response_code"]
|
188
|
+
===== `response_code`
|
189
|
+
|
190
|
+
* Value can be any of: 200, 201, 202, 204
|
191
|
+
* Default value is `200`
|
192
|
+
|
193
|
+
The HTTP return code if the request is processed successfully.
|
194
|
+
|
195
|
+
Other return codes may happen in the case of an error condition, such as
|
196
|
+
invalid credentials (401), internal errors (503) or backpressure (429).
|
197
|
+
|
198
|
+
If 204 (No Content) is set, the response body will not be sent in the response.
|
199
|
+
|
200
|
+
[id="plugins-{type}s-{plugin}-response_headers"]
|
201
|
+
===== `response_headers`
|
202
|
+
|
203
|
+
* Value type is <<hash,hash>>
|
204
|
+
* Default value is `{"Content-Type"=>"text/plain"}`
|
205
|
+
|
206
|
+
specify a custom set of response headers
|
207
|
+
|
208
|
+
[id="plugins-{type}s-{plugin}-remote_host_target_field"]
|
209
|
+
===== `remote_host_target_field`
|
210
|
+
|
211
|
+
* Value type is <<string,string>>
|
212
|
+
* Default value is `"host"`
|
213
|
+
|
214
|
+
specify a target field for the client host of the http request
|
215
|
+
|
216
|
+
[id="plugins-{type}s-{plugin}-request_headers_target_field"]
|
217
|
+
===== `request_headers_target_field`
|
218
|
+
|
219
|
+
* Value type is <<string,string>>
|
220
|
+
* Default value is `"headers"`
|
221
|
+
|
222
|
+
specify target field for the client host of the http request
|
223
|
+
|
224
|
+
[id="plugins-{type}s-{plugin}-ssl"]
|
225
|
+
===== `ssl`
|
226
|
+
|
227
|
+
* Value type is <<boolean,boolean>>
|
228
|
+
* Default value is `false`
|
229
|
+
|
230
|
+
Events are by default sent in plain text. You can
|
231
|
+
enable encryption by setting `ssl` to true and configuring
|
232
|
+
the `ssl_certificate` and `ssl_key` options.
|
233
|
+
|
234
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
235
|
+
===== `ssl_certificate`
|
236
|
+
|
237
|
+
* Value type is <<path,path>>
|
238
|
+
* There is no default value for this setting.
|
239
|
+
|
240
|
+
SSL certificate to use.
|
241
|
+
|
242
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
243
|
+
===== `ssl_certificate_authorities`
|
244
|
+
|
245
|
+
* Value type is <<array,array>>
|
246
|
+
* Default value is `[]`
|
247
|
+
|
248
|
+
Validate client certificates against these authorities.
|
249
|
+
You can define multiple files or paths. All the certificates will
|
250
|
+
be read and added to the trust store. You need to configure the `ssl_verify_mode`
|
251
|
+
to `peer` or `force_peer` to enable the verification.
|
252
|
+
|
253
|
+
|
254
|
+
[id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
|
255
|
+
===== `ssl_handshake_timeout`
|
256
|
+
|
257
|
+
* Value type is <<number,number>>
|
258
|
+
* Default value is `10000`
|
259
|
+
|
260
|
+
Time in milliseconds for an incomplete ssl handshake to timeout
|
261
|
+
|
262
|
+
[id="plugins-{type}s-{plugin}-ssl_key"]
|
263
|
+
===== `ssl_key`
|
264
|
+
|
265
|
+
* Value type is <<path,path>>
|
266
|
+
* There is no default value for this setting.
|
267
|
+
|
268
|
+
SSL key to use.
|
269
|
+
NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.0/apps/pkcs8.html[OpenSSL]
|
270
|
+
for more information.
|
271
|
+
|
272
|
+
[id="plugins-{type}s-{plugin}-ssl_key_passphrase"]
|
273
|
+
===== `ssl_key_passphrase`
|
274
|
+
|
275
|
+
* Value type is <<password,password>>
|
276
|
+
* There is no default value for this setting.
|
277
|
+
|
278
|
+
SSL key passphrase to use.
|
279
|
+
|
280
|
+
[id="plugins-{type}s-{plugin}-ssl_verify_mode"]
|
281
|
+
===== `ssl_verify_mode`
|
282
|
+
|
283
|
+
* Value can be any of: `none`, `peer`, `force_peer`
|
284
|
+
* Default value is `"none"`
|
285
|
+
|
286
|
+
By default the server doesn't do any client verification.
|
287
|
+
|
288
|
+
`peer` will make the server ask the client to provide a certificate.
|
289
|
+
If the client provides a certificate, it will be validated.
|
290
|
+
|
291
|
+
`force_peer` will make the server ask the client to provide a certificate.
|
292
|
+
If the client doesn't provide a certificate, the connection will be closed.
|
293
|
+
|
294
|
+
This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
|
295
|
+
|
296
|
+
[id="plugins-{type}s-{plugin}-threads"]
|
297
|
+
===== `threads`
|
298
|
+
|
299
|
+
* Value type is <<number,number>>
|
300
|
+
* Default value is number of processors
|
301
|
+
|
302
|
+
Number of threads to use for both accepting connections and handling requests
|
303
|
+
|
304
|
+
[id="plugins-{type}s-{plugin}-tls_max_version"]
|
305
|
+
===== `tls_max_version`
|
306
|
+
|
307
|
+
* Value type is <<number,number>>
|
308
|
+
* Default value is `1.2`
|
309
|
+
|
310
|
+
The maximum TLS version allowed for the encrypted connections. The value must be the one of the following:
|
311
|
+
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
312
|
+
|
313
|
+
[id="plugins-{type}s-{plugin}-tls_min_version"]
|
314
|
+
===== `tls_min_version`
|
315
|
+
|
316
|
+
* Value type is <<number,number>>
|
317
|
+
* Default value is `1`
|
318
|
+
|
319
|
+
The minimum TLS version allowed for the encrypted connections. The value must be one of the following:
|
320
|
+
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
321
|
+
|
322
|
+
[id="plugins-{type}s-{plugin}-user"]
|
323
|
+
===== `user`
|
324
|
+
|
325
|
+
* Value type is <<string,string>>
|
326
|
+
* There is no default value for this setting.
|
327
|
+
|
328
|
+
Username for basic authorization
|
329
|
+
|
330
|
+
[id="plugins-{type}s-{plugin}-verify_mode"]
|
331
|
+
===== `verify_mode`
|
332
|
+
|
333
|
+
* Value can be any of: `none`, `peer`, `force_peer`
|
334
|
+
* Default value is `"none"`
|
335
|
+
* This option is deprecated
|
336
|
+
|
337
|
+
Set the client certificate verification method. Valid methods: none, peer, force_peer
|
338
|
+
|
339
|
+
Note: This option is deprecated and it will be removed in the next major version of Logstash.
|
340
|
+
Use `ssl_verify_mode` instead.
|
341
|
+
|
342
|
+
|
343
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
344
|
+
include::{include_path}/{type}.asciidoc[]
|
345
|
+
|
346
|
+
:default_codec!:
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
|
2
|
+
|
3
|
+
require 'jar_dependencies'
|
4
|
+
require_jar('io.netty', 'netty-all', '4.1.49.Final')
|
5
|
+
require_jar('org.apache.logging.log4j', 'log4j-api', '2.11.1')
|
6
|
+
require_jar('com.google.protobuf', 'protobuf-java', '3.14.0')
|
7
|
+
require_jar('org.xerial.snappy', 'snappy-java', '1.1.8.4')
|
8
|
+
require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.12.3')
|
9
|
+
require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.12.3')
|
10
|
+
require_jar('org.logstash.plugins.input.http', 'logstash-input-http', '3.3.7')
|
@@ -0,0 +1,316 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/inputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require "stud/interval"
|
5
|
+
require "logstash-input-http_jars"
|
6
|
+
|
7
|
+
# Using this input you can receive single or multiline events over http(s).
|
8
|
+
# Applications can send a HTTP POST request with a body to the endpoint started by this
|
9
|
+
# input and Logstash will convert it into an event for subsequent processing. Users
|
10
|
+
# can pass plain text, JSON, or any formatted data and use a corresponding codec with this
|
11
|
+
# input. For Content-Type `application/json` the `json` codec is used, but for all other
|
12
|
+
# data formats, `plain` codec is used.
|
13
|
+
#
|
14
|
+
# This input can also be used to receive webhook requests to integrate with other services
|
15
|
+
# and applications. By taking advantage of the vast plugin ecosystem available in Logstash
|
16
|
+
# you can trigger actionable events right from your application.
|
17
|
+
#
|
18
|
+
# ==== Security
|
19
|
+
# This plugin supports standard HTTP basic authentication headers to identify the requester.
|
20
|
+
# You can pass in an username, password combination while sending data to this input
|
21
|
+
#
|
22
|
+
# You can also setup SSL and send data securely over https, with an option of validating
|
23
|
+
# the client's certificate. Currently, the certificate setup is through
|
24
|
+
# https://docs.oracle.com/cd/E19509-01/820-3503/ggfen/index.html[Java Keystore
|
25
|
+
# format]
|
26
|
+
#
|
27
|
+
class LogStash::Inputs::Http < LogStash::Inputs::Base
|
28
|
+
require "logstash/inputs/http/tls"
|
29
|
+
|
30
|
+
java_import "io.netty.handler.codec.http.HttpUtil"
|
31
|
+
|
32
|
+
config_name "http"
|
33
|
+
|
34
|
+
# Codec used to decode the incoming data.
|
35
|
+
# This codec will be used as a fall-back if the content-type
|
36
|
+
# is not found in the "additional_codecs" hash
|
37
|
+
default :codec, "plain"
|
38
|
+
|
39
|
+
# The host or ip to bind
|
40
|
+
config :host, :validate => :string, :default => "0.0.0.0"
|
41
|
+
|
42
|
+
# The TCP port to bind to
|
43
|
+
config :port, :validate => :number, :default => 8080
|
44
|
+
|
45
|
+
# Username for basic authorization
|
46
|
+
config :user, :validate => :string, :required => false
|
47
|
+
|
48
|
+
# Password for basic authorization
|
49
|
+
config :password, :validate => :password, :required => false
|
50
|
+
|
51
|
+
# Events are by default sent in plain text. You can
|
52
|
+
# enable encryption by setting `ssl` to true and configuring
|
53
|
+
# the `ssl_certificate` and `ssl_key` options.
|
54
|
+
config :ssl, :validate => :boolean, :default => false
|
55
|
+
|
56
|
+
# SSL certificate to use.
|
57
|
+
config :ssl_certificate, :validate => :path
|
58
|
+
|
59
|
+
# SSL key to use.
|
60
|
+
# NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.0/apps/pkcs8.html[OpenSSL]
|
61
|
+
# for more information.
|
62
|
+
config :ssl_key, :validate => :path
|
63
|
+
|
64
|
+
# SSL key passphrase to use.
|
65
|
+
config :ssl_key_passphrase, :validate => :password
|
66
|
+
|
67
|
+
# Validate client certificates against these authorities.
|
68
|
+
# You can define multiple files or paths. All the certificates will
|
69
|
+
# be read and added to the trust store. You need to configure the `ssl_verify_mode`
|
70
|
+
# to `peer` or `force_peer` to enable the verification.
|
71
|
+
config :ssl_certificate_authorities, :validate => :array, :default => []
|
72
|
+
|
73
|
+
# By default the server doesn't do any client verification.
|
74
|
+
#
|
75
|
+
# `peer` will make the server ask the client to provide a certificate.
|
76
|
+
# If the client provides a certificate, it will be validated.
|
77
|
+
#
|
78
|
+
# `force_peer` will make the server ask the client to provide a certificate.
|
79
|
+
# If the client doesn't provide a certificate, the connection will be closed.
|
80
|
+
#
|
81
|
+
# This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
|
82
|
+
config :ssl_verify_mode, :validate => ["none", "peer", "force_peer"], :default => "none"
|
83
|
+
|
84
|
+
# Time in milliseconds for an incomplete ssl handshake to timeout
|
85
|
+
config :ssl_handshake_timeout, :validate => :number, :default => 10000
|
86
|
+
|
87
|
+
# The minimum TLS version allowed for the encrypted connections. The value must be one of the following:
|
88
|
+
# 1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
89
|
+
config :tls_min_version, :validate => :number, :default => TLS.min.version
|
90
|
+
|
91
|
+
# The maximum TLS version allowed for the encrypted connections. The value must be the one of the following:
|
92
|
+
# 1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
93
|
+
config :tls_max_version, :validate => :number, :default => TLS.max.version
|
94
|
+
|
95
|
+
# The list of ciphers suite to use, listed by priorities.
|
96
|
+
config :cipher_suites, :validate => :array, :default => org.logstash.plugins.inputs.http.util.SslSimpleBuilder.getDefaultCiphers
|
97
|
+
|
98
|
+
# Apply specific codecs for specific content types.
|
99
|
+
# The default codec will be applied only after this list is checked
|
100
|
+
# and no codec for the request's content-type is found
|
101
|
+
config :additional_codecs, :validate => :hash, :default => { "application/json" => "json" }
|
102
|
+
|
103
|
+
# specify a custom set of response headers
|
104
|
+
config :response_headers, :validate => :hash, :default => { 'Content-Type' => 'text/plain' }
|
105
|
+
|
106
|
+
# target field for the client host of the http request
|
107
|
+
config :remote_host_target_field, :validate => :string, :default => "host"
|
108
|
+
|
109
|
+
# target field for the client host of the http request
|
110
|
+
config :request_headers_target_field, :validate => :string, :default => "headers"
|
111
|
+
|
112
|
+
config :threads, :validate => :number, :required => false, :default => ::LogStash::Config::CpuCoreStrategy.maximum
|
113
|
+
|
114
|
+
config :max_pending_requests, :validate => :number, :required => false, :default => 200
|
115
|
+
|
116
|
+
config :max_content_length, :validate => :number, :required => false, :default => 100 * 1024 * 1024
|
117
|
+
|
118
|
+
config :response_code, :validate => [200, 201, 202, 204], :default => 200
|
119
|
+
# Deprecated options
|
120
|
+
|
121
|
+
# The JKS keystore to validate the client's certificates
|
122
|
+
config :keystore, :validate => :path, :deprecated => "Set 'ssl_certificate' and 'ssl_key' instead."
|
123
|
+
config :keystore_password, :validate => :password, :deprecated => "Set 'ssl_key_passphrase' instead."
|
124
|
+
|
125
|
+
config :verify_mode, :validate => ['none', 'peer', 'force_peer'], :default => 'none',
|
126
|
+
:deprecated => "Set 'ssl_verify_mode' instead."
|
127
|
+
|
128
|
+
public
|
129
|
+
def register
|
130
|
+
|
131
|
+
validate_ssl_settings!
|
132
|
+
|
133
|
+
if @user && @password then
|
134
|
+
token = Base64.strict_encode64("#{@user}:#{@password.value}")
|
135
|
+
@auth_token = "Basic #{token}"
|
136
|
+
end
|
137
|
+
|
138
|
+
@codecs = Hash.new
|
139
|
+
|
140
|
+
@additional_codecs.each do |content_type, codec|
|
141
|
+
@codecs[content_type] = LogStash::Plugin.lookup("codec", codec).new
|
142
|
+
end
|
143
|
+
|
144
|
+
|
145
|
+
@promtail_input = org.logstash.plugins.inputs.http.promtail.PromtailHandler.new
|
146
|
+
|
147
|
+
require "logstash/inputs/http/message_handler"
|
148
|
+
message_handler = MessageHandler.new(self, @codec, @codecs, @auth_token)
|
149
|
+
@http_server = create_http_server(message_handler)
|
150
|
+
end # def register
|
151
|
+
|
152
|
+
def run(queue)
|
153
|
+
@queue = queue
|
154
|
+
@logger.info("Starting http input listener", :address => "#{@host}:#{@port}", :ssl => "#{@ssl}")
|
155
|
+
@http_server.run()
|
156
|
+
end
|
157
|
+
|
158
|
+
def stop
|
159
|
+
@http_server.close() rescue nil
|
160
|
+
end
|
161
|
+
|
162
|
+
def close
|
163
|
+
@http_server.close() rescue nil
|
164
|
+
end
|
165
|
+
|
166
|
+
def decode_body(headers, remote_address, body, default_codec, additional_codecs)
|
167
|
+
content_type = headers.fetch("content_type", "")
|
168
|
+
|
169
|
+
if (content_type.start_with?("application/x-protobuf"))
|
170
|
+
# events = @promtail_input.decode_str(@promtail_input.toUTF8String(body))
|
171
|
+
events = @promtail_input.decode(body)
|
172
|
+
events.each do |event|
|
173
|
+
push_decoded_event(headers, remote_address, LogStash::Event.new(event), false)
|
174
|
+
end
|
175
|
+
else
|
176
|
+
body_str = body
|
177
|
+
body_str = @promtail_input.toUTF8String(body) if !body.is_a?(String)
|
178
|
+
codec = additional_codecs.fetch(HttpUtil.getMimeType(content_type), default_codec)
|
179
|
+
codec.decode(body_str) { |event| push_decoded_event(headers, remote_address, event) }
|
180
|
+
codec.flush { |event| push_decoded_event(headers, remote_address, event) }
|
181
|
+
end
|
182
|
+
|
183
|
+
true
|
184
|
+
rescue => e
|
185
|
+
@logger.error(
|
186
|
+
"unable to process event.",
|
187
|
+
:message => e.message,
|
188
|
+
:class => e.class.name,
|
189
|
+
:backtrace => e.backtrace
|
190
|
+
)
|
191
|
+
false
|
192
|
+
end
|
193
|
+
|
194
|
+
def push_decoded_event(headers, remote_address, event, add_headers=true)
|
195
|
+
if add_headers
|
196
|
+
event.set(@request_headers_target_field, headers)
|
197
|
+
event.set(@remote_host_target_field, remote_address)
|
198
|
+
end
|
199
|
+
tenant = headers.fetch("tenant", "")
|
200
|
+
tenant = headers.fetch("x_scope_orgid", "") if tenant.empty?
|
201
|
+
if !tenant.empty?
|
202
|
+
event.set("tenant", tenant)
|
203
|
+
end
|
204
|
+
decorate(event)
|
205
|
+
@logger.debug("Pushing request to #{remote_address} with headers #{headers} event: #{event}")
|
206
|
+
@queue << event
|
207
|
+
end
|
208
|
+
|
209
|
+
def validate_ssl_settings!
|
210
|
+
if !@ssl
|
211
|
+
@logger.warn("SSL Certificate will not be used") if @ssl_certificate
|
212
|
+
@logger.warn("SSL Key will not be used") if @ssl_key
|
213
|
+
@logger.warn("SSL Java Key Store will not be used") if @keystore
|
214
|
+
elsif !(ssl_key_configured? || ssl_jks_configured?)
|
215
|
+
raise LogStash::ConfigurationError, "Certificate or JKS must be configured"
|
216
|
+
end
|
217
|
+
|
218
|
+
if @ssl && (original_params.key?("verify_mode") && original_params.key?("ssl_verify_mode"))
|
219
|
+
raise LogStash::ConfigurationError, "Both 'ssl_verify_mode' and 'verify_mode' were set. Use only 'ssl_verify_mode'."
|
220
|
+
elsif original_params.key?("verify_mode")
|
221
|
+
@ssl_verify_mode_final = @verify_mode
|
222
|
+
elsif original_params.key?("ssl_verify_mode")
|
223
|
+
@ssl_verify_mode_final = @ssl_verify_mode
|
224
|
+
else
|
225
|
+
@ssl_verify_mode_final = @ssl_verify_mode
|
226
|
+
end
|
227
|
+
|
228
|
+
if @ssl && require_certificate_authorities? && !client_authentication?
|
229
|
+
raise LogStash::ConfigurationError, "Using `ssl_verify_mode` or `verify_mode` set to PEER or FORCE_PEER, requires the configuration of `ssl_certificate_authorities`"
|
230
|
+
elsif @ssl && !require_certificate_authorities? && client_authentication?
|
231
|
+
raise LogStash::ConfigurationError, "The configuration of `ssl_certificate_authorities` requires setting `ssl_verify_mode` or `verify_mode` to PEER or FORCE_PEER"
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def create_http_server(message_handler)
|
236
|
+
org.logstash.plugins.inputs.http.NettyHttpServer.new(
|
237
|
+
@host, @port, message_handler, build_ssl_params(), @threads, @max_pending_requests, @max_content_length, @response_code)
|
238
|
+
end
|
239
|
+
|
240
|
+
def build_ssl_params
|
241
|
+
return nil unless @ssl
|
242
|
+
|
243
|
+
if @keystore && @keystore_password
|
244
|
+
ssl_builder = org.logstash.plugins.inputs.http.util.JksSslBuilder.new(@keystore, @keystore_password.value)
|
245
|
+
else
|
246
|
+
begin
|
247
|
+
ssl_builder = org.logstash.plugins.inputs.http.util.SslSimpleBuilder
|
248
|
+
.new(@ssl_certificate, @ssl_key, @ssl_key_passphrase.nil? ? nil : @ssl_key_passphrase.value)
|
249
|
+
.setCipherSuites(normalized_ciphers)
|
250
|
+
rescue java.lang.IllegalArgumentException => e
|
251
|
+
@logger.error("SSL configuration invalid", error_details(e))
|
252
|
+
raise LogStash::ConfigurationError, e
|
253
|
+
end
|
254
|
+
|
255
|
+
if client_authentication?
|
256
|
+
ssl_builder.setCertificateAuthorities(@ssl_certificate_authorities)
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
new_ssl_handshake_provider(ssl_builder)
|
261
|
+
end
|
262
|
+
|
263
|
+
def ssl_key_configured?
|
264
|
+
!!(@ssl_certificate && @ssl_key)
|
265
|
+
end
|
266
|
+
|
267
|
+
def ssl_jks_configured?
|
268
|
+
!!(@keystore && @keystore_password)
|
269
|
+
end
|
270
|
+
|
271
|
+
def client_authentication?
|
272
|
+
@ssl_certificate_authorities && @ssl_certificate_authorities.size > 0
|
273
|
+
end
|
274
|
+
|
275
|
+
def require_certificate_authorities?
|
276
|
+
@ssl_verify_mode_final == "force_peer" || @ssl_verify_mode_final == "peer"
|
277
|
+
end
|
278
|
+
|
279
|
+
private
|
280
|
+
|
281
|
+
def normalized_ciphers
|
282
|
+
@cipher_suites.map(&:upcase)
|
283
|
+
end
|
284
|
+
|
285
|
+
def convert_protocols
|
286
|
+
TLS.get_supported(@tls_min_version..@tls_max_version).map(&:name)
|
287
|
+
end
|
288
|
+
|
289
|
+
def new_ssl_handshake_provider(ssl_builder)
|
290
|
+
begin
|
291
|
+
ssl_handler_provider = org.logstash.plugins.inputs.http.util.SslHandlerProvider.new(ssl_builder.build())
|
292
|
+
ssl_handler_provider.setVerifyMode(@ssl_verify_mode_final.upcase)
|
293
|
+
ssl_handler_provider.setProtocols(convert_protocols)
|
294
|
+
ssl_handler_provider.setHandshakeTimeoutMilliseconds(@ssl_handshake_timeout)
|
295
|
+
ssl_handler_provider
|
296
|
+
rescue java.lang.IllegalArgumentException => e
|
297
|
+
@logger.error("SSL configuration invalid", error_details(e))
|
298
|
+
raise LogStash::ConfigurationError, e
|
299
|
+
rescue java.lang.Exception => e
|
300
|
+
@logger.error("SSL configuration failed", error_details(e, true))
|
301
|
+
raise e
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
def error_details(e, trace = false)
|
306
|
+
error_details = { :exception => e.class, :message => e.message }
|
307
|
+
error_details[:backtrace] = e.backtrace if trace || @logger.debug?
|
308
|
+
cause = e.cause
|
309
|
+
if cause && e != cause
|
310
|
+
error_details[:cause] = { :exception => cause.class, :message => cause.message }
|
311
|
+
error_details[:cause][:backtrace] = cause.backtrace if trace || @logger.debug?
|
312
|
+
end
|
313
|
+
error_details
|
314
|
+
end
|
315
|
+
|
316
|
+
end # class LogStash::Inputs::Http
|