logstash-integration-logstash 0.0.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +4 -0
- data/Gemfile +11 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +2 -0
- data/README.md +98 -0
- data/VERSION +1 -0
- data/docs/index.asciidoc +71 -0
- data/docs/input-logstash.asciidoc +252 -0
- data/docs/output-logstash.asciidoc +271 -0
- data/lib/logstash/inputs/logstash.rb +185 -0
- data/lib/logstash/outputs/logstash.rb +162 -0
- data/logstash-integration-logstash.gemspec +36 -0
- data/spec/fixtures/certs/generate.sh +69 -0
- data/spec/fixtures/certs/generated/README.txt +2 -0
- data/spec/fixtures/certs/generated/client_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_root.key.pem +52 -0
- data/spec/fixtures/certs/generated/client_from_root.key.pkcs8.pem +54 -0
- data/spec/fixtures/certs/generated/client_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_root.pem +35 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pem +52 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pkcs8.pem +54 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.pem +35 -0
- data/spec/fixtures/certs/generated/client_self_signed.jks +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.key.pem +52 -0
- data/spec/fixtures/certs/generated/client_self_signed.key.pkcs8.pem +54 -0
- data/spec/fixtures/certs/generated/client_self_signed.p12 +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.pem +32 -0
- data/spec/fixtures/certs/generated/root.key.pem +52 -0
- data/spec/fixtures/certs/generated/root.pem +32 -0
- data/spec/fixtures/certs/generated/server_from_root-key-pkcs8.pem +52 -0
- data/spec/fixtures/certs/generated/server_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/server_from_root.key.pem +52 -0
- data/spec/fixtures/certs/generated/server_from_root.key.pkcs8.pem +54 -0
- data/spec/fixtures/certs/generated/server_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/server_from_root.pem +37 -0
- data/spec/fixtures/certs/generated/untrusted.key.pem +52 -0
- data/spec/fixtures/certs/generated/untrusted.pem +32 -0
- data/spec/fixtures/certs/openssl.cnf +57 -0
- data/spec/spec_helper.rb +22 -0
- data/spec/unit/full_transmission_spec.rb +202 -0
- data/spec/unit/logstash_input_spec.rb +151 -0
- data/spec/unit/logstash_output_spec.rb +170 -0
- metadata +243 -0
@@ -0,0 +1,271 @@
|
|
1
|
+
:integration: logstash
|
2
|
+
:plugin: logstash
|
3
|
+
:type: output
|
4
|
+
:no_codec:
|
5
|
+
|
6
|
+
///////////////////////////////////////////
|
7
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
8
|
+
///////////////////////////////////////////
|
9
|
+
:version: %VERSION%
|
10
|
+
:release_date: %RELEASE_DATE%
|
11
|
+
:changelog_url: %CHANGELOG_URL%
|
12
|
+
:include_path: ../../../../logstash/docs/include
|
13
|
+
///////////////////////////////////////////
|
14
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
15
|
+
///////////////////////////////////////////
|
16
|
+
|
17
|
+
[id="plugins-{type}s-{plugin}"]
|
18
|
+
|
19
|
+
=== Logstash output plugin
|
20
|
+
|
21
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
22
|
+
|
23
|
+
==== Description
|
24
|
+
|
25
|
+
Send events to a <<plugins-inputs-logstash>> in a pipeline that may be in another process or on another host.
|
26
|
+
You must have a TCP route to the port on an interface that the downstream input is bound to.
|
27
|
+
|
28
|
+
NOTE: Sending events to _any_ destination other than <<plugins-inputs-logstash>> is neither advised nor supported.
|
29
|
+
We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding.
|
30
|
+
|
31
|
+
[id="plugins-{type}s-{plugin}-minimum-config"]
|
32
|
+
===== Minimum Configuration
|
33
|
+
[cols="2a,2a"]
|
34
|
+
|=======================================================================================================================
|
35
|
+
|SSL Enabled |SSL Disabled
|
36
|
+
|
37
|
+
|
|
38
|
+
|
39
|
+
[source]
|
40
|
+
----
|
41
|
+
output {
|
42
|
+
logstash {
|
43
|
+
host => "10.0.0.123"
|
44
|
+
port => 8080
|
45
|
+
}
|
46
|
+
}
|
47
|
+
----
|
48
|
+
|
49
|
+
|
|
50
|
+
|
51
|
+
[source]
|
52
|
+
----
|
53
|
+
output {
|
54
|
+
logstash {
|
55
|
+
host => "10.0.0.123"
|
56
|
+
port => 8080
|
57
|
+
ssl_enabled
|
58
|
+
=> false
|
59
|
+
}
|
60
|
+
}
|
61
|
+
----
|
62
|
+
|
63
|
+
|=======================================================================================================================
|
64
|
+
|
65
|
+
[id="plugins-{type}s-{plugin}-config-connecting"]
|
66
|
+
===== Configuration Concepts
|
67
|
+
|
68
|
+
This output plugin needs to be configured to connect to a <<plugins-inputs-logstash>> by specifying its <<plugins-{type}s-{plugin}-host>> and <<plugins-{type}s-{plugin}-port>>.
|
69
|
+
Depending on the downstream plugin's configuration, you may need to also configure SSL and/or credentials.
|
70
|
+
|
71
|
+
[id="plugins-{type}s-{plugin}-config-ssl-trust"]
|
72
|
+
===== Security: SSL Trust
|
73
|
+
|
74
|
+
When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events.
|
75
|
+
|
76
|
+
It does so by ensuring that the server presents a currently-valid certificate with identity claims matching the <<plugins-{type}s-{plugin}-host>> we are configured to connect to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material.
|
77
|
+
|
78
|
+
The system trust store is used by default.
|
79
|
+
You can provide an _alternate_ source of trust with _ONE OF_:
|
80
|
+
|
81
|
+
* A PEM-formatted list of trusted certificate authorities (see <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>)
|
82
|
+
* A PKCS12- or JKS-formatted truststore (see <<plugins-{type}s-{plugin}-ssl_truststore_path>>)
|
83
|
+
|
84
|
+
[id="plugins-{type}s-{plugin}-config-ssl-identity"]
|
85
|
+
===== Security: SSL Identity
|
86
|
+
|
87
|
+
If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_:
|
88
|
+
|
89
|
+
* JKS- or PKCS12-formatted Keystore (see <<plugins-{type}s-{plugin}-ssl_keystore_path>>)
|
90
|
+
* PKCS8-formatted Certificate/Key pair (see <<plugins-{type}s-{plugin}-ssl_certificate>>)
|
91
|
+
|
92
|
+
[id="plugins-{type}s-{plugin}-config-credentials"]
|
93
|
+
===== Security: Credentials
|
94
|
+
|
95
|
+
If the downstream <<plugins-inputs-logstash>> is configured to require <<plugins-inputs-logstash-username>> and <<plugins-inputs-logstash-password>>,
|
96
|
+
you will need to configure this output with a matching <<plugins-{type}s-{plugin}-username>> and <<plugins-{type}s-{plugin}-password>>.
|
97
|
+
|
98
|
+
NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text.
|
99
|
+
|
100
|
+
[id="plugins-{type}s-{plugin}-options"]
|
101
|
+
==== Logstash Output Configuration Options
|
102
|
+
|
103
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
104
|
+
|
105
|
+
[cols="<,<,<",options="header",]
|
106
|
+
|=======================================================================
|
107
|
+
|Setting |Input type |Required
|
108
|
+
| <<plugins-{type}s-{plugin}-host>> |<<string,string>> |Yes
|
109
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
110
|
+
| <<plugins-{type}s-{plugin}-port>> |<<number,number>> |Yes
|
111
|
+
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
112
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
|
113
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
|
114
|
+
| <<plugins-{type}s-{plugin}-ssl_key>> | <<path,path>>|No
|
115
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_path>> | <<path,path>>|No
|
116
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_password>> | <<password,password>>|No
|
117
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_path>> | <<path,path>>|No
|
118
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_password>> | <<password,password>>|No
|
119
|
+
| <<plugins-{type}s-{plugin}-ssl_verification_mode>> | <<string,string>>, one of `["full", "none"]`|No
|
120
|
+
| <<plugins-{type}s-{plugin}-username>> |<<string,string>>|No
|
121
|
+
|=======================================================================
|
122
|
+
|
123
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
124
|
+
output plugins.
|
125
|
+
|
126
|
+
|
127
|
+
|
128
|
+
[id="plugins-{type}s-{plugin}-host"]
|
129
|
+
===== `host`
|
130
|
+
|
131
|
+
* Value type is a <<string,string>> ip address or hostname
|
132
|
+
* There is no default value
|
133
|
+
|
134
|
+
Specify which host to connect to by providing its ip address or resolvable hostname.
|
135
|
+
|
136
|
+
NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address.
|
137
|
+
|
138
|
+
[id="plugins-{type}s-{plugin}-port"]
|
139
|
+
===== `port`
|
140
|
+
|
141
|
+
* Value type is a <<number,number>> port
|
142
|
+
* There is no default value
|
143
|
+
|
144
|
+
Specify which port to connect to.
|
145
|
+
|
146
|
+
[id="plugins-{type}s-{plugin}-password"]
|
147
|
+
===== `password`
|
148
|
+
|
149
|
+
* Value type is <<password,password>>
|
150
|
+
* There is no default value for this setting.
|
151
|
+
* Required when <<plugins-{type}s-{plugin}-username>> is configured.
|
152
|
+
|
153
|
+
Password for password-based authentication.
|
154
|
+
|
155
|
+
When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
|
156
|
+
|
157
|
+
[id="plugins-{type}s-{plugin}-ssl_enabled"]
|
158
|
+
===== `ssl_enabled`
|
159
|
+
|
160
|
+
* Value type is <<boolean,boolean>>
|
161
|
+
* Default value is `true`
|
162
|
+
|
163
|
+
Logstash-to-Logstash communication is secured by default.
|
164
|
+
When the downstream input plugin disables SSL, it must also be disabled here.
|
165
|
+
|
166
|
+
You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure.
|
167
|
+
|
168
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
169
|
+
===== `ssl_certificate`
|
170
|
+
|
171
|
+
* Value type is <<path,path>>
|
172
|
+
* There is no default value for this setting.
|
173
|
+
* When present, <<plugins-{type}s-{plugin}-ssl_key>> is also required.
|
174
|
+
* Cannot be combined with configurations that disable SSL.
|
175
|
+
|
176
|
+
Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input.
|
177
|
+
|
178
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
179
|
+
===== `ssl_certificate_authorities`
|
180
|
+
|
181
|
+
* Value type is a <<path,path>>
|
182
|
+
* There is no default value for this setting.
|
183
|
+
* Cannot be combined with configurations that disable SSL.
|
184
|
+
* Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
|
185
|
+
|
186
|
+
One or more PEM-encoded files defining certificate authorities for use in downstream input authentication.
|
187
|
+
This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input.
|
188
|
+
|
189
|
+
[id="plugins-{type}s-{plugin}-ssl_key"]
|
190
|
+
===== `ssl_key`
|
191
|
+
|
192
|
+
* Value type is <<path,path>>
|
193
|
+
* There is no default value for this setting.
|
194
|
+
* Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_certificate>>
|
195
|
+
* Cannot be combined with configurations that disable SSL.
|
196
|
+
|
197
|
+
A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key.
|
198
|
+
|
199
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
|
200
|
+
===== `ssl_keystore_path`
|
201
|
+
|
202
|
+
* Value type is <<path,path>>
|
203
|
+
* There is no default value for this setting.
|
204
|
+
* When present, <<plugins-{type}s-{plugin}-ssl_keystore_password>> is also required.
|
205
|
+
* Cannot be combined with configurations that disable SSL.
|
206
|
+
|
207
|
+
A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input.
|
208
|
+
The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
|
209
|
+
|
210
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
|
211
|
+
===== `ssl_keystore_password`
|
212
|
+
|
213
|
+
* Value type is <<password,password>>
|
214
|
+
* There is no default value for this setting.
|
215
|
+
* Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_keystore_path>>
|
216
|
+
* Cannot be combined with configurations that disable SSL.
|
217
|
+
|
218
|
+
Password for the <<plugins-{type}s-{plugin}-ssl_keystore_path>>
|
219
|
+
|
220
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
|
221
|
+
===== `ssl_truststore_path`
|
222
|
+
|
223
|
+
* Value type is <<path,path>>
|
224
|
+
* There is no default value for this setting.
|
225
|
+
* When present, <<plugins-{type}s-{plugin}-ssl_truststore_path>> is also required.
|
226
|
+
* Cannot be combined with configurations that disable SSL.
|
227
|
+
* Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
|
228
|
+
|
229
|
+
A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input.
|
230
|
+
The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
|
231
|
+
|
232
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
|
233
|
+
===== `ssl_truststore_password`
|
234
|
+
|
235
|
+
* Value type is <<password,password>>
|
236
|
+
* There is no default value for this setting.
|
237
|
+
* Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_truststore_path>>
|
238
|
+
* Cannot be combined with configurations that disable SSL.
|
239
|
+
|
240
|
+
Password for the <<plugins-{type}s-{plugin}-ssl_truststore_path>>
|
241
|
+
|
242
|
+
[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
|
243
|
+
===== `ssl_verification_mode`
|
244
|
+
|
245
|
+
* Value type is <<string,string>>
|
246
|
+
* The supported modes are:
|
247
|
+
** `full`: verifies that a certificate provided by the client has an identity claim matching <<plugins-{type}s-{plugin}-host>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key.
|
248
|
+
** `none`: performs no validation of the presented certificate
|
249
|
+
|
250
|
+
* The default value is `full`.
|
251
|
+
* Cannot be combined with configurations that disable SSL.
|
252
|
+
|
253
|
+
When communicating over SSL, this setting controls how the downstream input's certificate is verified.
|
254
|
+
|
255
|
+
[id="plugins-{type}s-{plugin}-username"]
|
256
|
+
===== `username`
|
257
|
+
|
258
|
+
* Value type is <<string,string>>
|
259
|
+
* There is no default value for this setting.
|
260
|
+
* When present, <<plugins-{type}s-{plugin}-password>> is also required.
|
261
|
+
|
262
|
+
Username for password-based authentication.
|
263
|
+
|
264
|
+
When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
|
265
|
+
|
266
|
+
NOTE: when SSL is disabled, credentials will be transmitted in clear-text.
|
267
|
+
|
268
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
269
|
+
include::{include_path}/{type}.asciidoc[]
|
270
|
+
|
271
|
+
:default_codec!:
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'logstash/inputs/base'
|
4
|
+
require 'logstash/namespace'
|
5
|
+
|
6
|
+
require "logstash/plugin_mixins/plugin_factory_support"
|
7
|
+
|
8
|
+
require 'logstash/codecs/json_lines'
|
9
|
+
|
10
|
+
class LogStash::Inputs::Logstash < LogStash::Inputs::Base
|
11
|
+
include LogStash::PluginMixins::PluginFactorySupport
|
12
|
+
|
13
|
+
config_name "logstash"
|
14
|
+
|
15
|
+
config :host, :validate => :string, :default => "0.0.0.0"
|
16
|
+
config :port, :validate => :number, :required => true
|
17
|
+
|
18
|
+
# optional username/password credentials
|
19
|
+
config :username, :validate => :string, :required => false
|
20
|
+
config :password, :validate => :password, :required => false
|
21
|
+
|
22
|
+
config :ssl_enabled, :validate => :boolean, :default => true
|
23
|
+
|
24
|
+
# SSL:IDENTITY:SOURCE cert/key pair
|
25
|
+
config :ssl_certificate, :validate => :path
|
26
|
+
config :ssl_key, :validate => :path
|
27
|
+
config :ssl_key_passphrase, :validate => :password
|
28
|
+
|
29
|
+
# SSL:IDENTITY:SOURCE keystore
|
30
|
+
config :ssl_keystore_path, :validate => :path
|
31
|
+
config :ssl_keystore_password, :validate => :password
|
32
|
+
|
33
|
+
# SSL:TRUST:CONFIG
|
34
|
+
config :ssl_client_authentication, :validate => %w(none optional required), :default => 'none'
|
35
|
+
|
36
|
+
# SSL:TRUST:SOURCE ca file
|
37
|
+
config :ssl_certificate_authorities, :validate => :path, :list => true
|
38
|
+
|
39
|
+
# SSL:TUNING
|
40
|
+
config :ssl_handshake_timeout, :validate => :number, default: 10_000
|
41
|
+
config :ssl_cipher_suites, :validate => :string, :list => true
|
42
|
+
config :ssl_supported_protocols, :validate => :string, :list => true
|
43
|
+
|
44
|
+
def initialize(*a)
|
45
|
+
super
|
46
|
+
|
47
|
+
if original_params.include?('codec')
|
48
|
+
report_invalid_config! 'The `logstash` input does not have an externally-configurable `codec`'
|
49
|
+
end
|
50
|
+
|
51
|
+
logger.debug("initializing inner HTTP input plugin")
|
52
|
+
@internal_http = plugin_factory.input('http').new(inner_http_input_options)
|
53
|
+
logger.debug("inner HTTP input plugin has been initialized")
|
54
|
+
end
|
55
|
+
|
56
|
+
def register
|
57
|
+
logger.debug("registering inner HTTP input plugin")
|
58
|
+
@internal_http.register
|
59
|
+
logger.debug("inner HTTP input plugin has been registered")
|
60
|
+
end
|
61
|
+
|
62
|
+
def run(queue)
|
63
|
+
logger.debug("starting inner HTTP input plugin")
|
64
|
+
@internal_http.run(QueueWrapper.new(queue))
|
65
|
+
logger.debug("inner HTTP input plugin has exited normally")
|
66
|
+
rescue => e
|
67
|
+
logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
|
68
|
+
raise
|
69
|
+
end
|
70
|
+
|
71
|
+
def stop
|
72
|
+
logger.debug("stopping inner HTTP input plugin")
|
73
|
+
@internal_http.stop
|
74
|
+
logger.debug('inner HTTP plugin has been stopped')
|
75
|
+
end
|
76
|
+
|
77
|
+
def close
|
78
|
+
logger.debug("closing inner HTTP input plugin")
|
79
|
+
@internal_http.close
|
80
|
+
logger.debug('inner HTTP plugin has been closed')
|
81
|
+
end
|
82
|
+
|
83
|
+
def inner_http_input_options
|
84
|
+
@_inner_http_input_options ||= begin
|
85
|
+
http_options = {
|
86
|
+
# directly-configurable
|
87
|
+
'host' => @host,
|
88
|
+
'port' => @port,
|
89
|
+
|
90
|
+
# non-configurable codec
|
91
|
+
'codec' => plugin_factory.codec('json_lines').new(inner_json_lines_codec_options),
|
92
|
+
'additional_codecs' => {},
|
93
|
+
'response_headers' => { 'Accept' => 'application/x-ndjson' },
|
94
|
+
|
95
|
+
# enrichment avoidance
|
96
|
+
'ecs_compatibility' => 'disabled',
|
97
|
+
'remote_host_target_field' => '[@metadata][void]',
|
98
|
+
'request_headers_target_field' => '[@metadata][void]',
|
99
|
+
}
|
100
|
+
|
101
|
+
if @username
|
102
|
+
http_options['user'] = @username
|
103
|
+
http_options['password'] = @password || report_invalid_config!('`password` is REQUIRED when `username` is provided')
|
104
|
+
logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
|
105
|
+
elsif @password
|
106
|
+
report_invalid_config!('`password` not allowed unless `username` is configured')
|
107
|
+
end
|
108
|
+
|
109
|
+
if @ssl_enabled == false
|
110
|
+
rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
|
111
|
+
report_invalid_config!("Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
|
112
|
+
else
|
113
|
+
http_options['ssl_enabled'] = true
|
114
|
+
|
115
|
+
http_options['ssl_cipher_suites'] = @ssl_cipher_suites if @original_params.include?('ssl_cipher_suites')
|
116
|
+
http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
|
117
|
+
http_options['ssl_handshake_timeout'] = @ssl_handshake_timeout
|
118
|
+
|
119
|
+
http_options.merge!(ssl_identity_options)
|
120
|
+
http_options.merge!(ssl_trust_options)
|
121
|
+
end
|
122
|
+
|
123
|
+
http_options
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def ssl_identity_options
|
128
|
+
{}.tap do |identity_options|
|
129
|
+
if @ssl_certificate && @ssl_keystore_path
|
130
|
+
report_invalid_config!('SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
|
131
|
+
elsif @ssl_certificate
|
132
|
+
identity_options['ssl_certificate'] = @ssl_certificate
|
133
|
+
identity_options['ssl_key'] = @ssl_key || report_invalid_config!('`ssl_key` is required when `ssl_certificate` is configured')
|
134
|
+
identity_options['ssl_key_passphrase'] = @ssl_key_passphrase unless @ssl_key_passphrase.nil?
|
135
|
+
elsif @ssl_key
|
136
|
+
report_invalid_config!('`ssl_key` is not allowed unless `ssl_certificate` is configured')
|
137
|
+
elsif @ssl_key_passphrase
|
138
|
+
report_invalid_config!('`ssl_key_passphrase` is not allowed unless `ssl_key` is configured')
|
139
|
+
elsif @ssl_keystore_path
|
140
|
+
identity_options['ssl_keystore_path'] = @ssl_keystore_path
|
141
|
+
identity_options['ssl_keystore_password'] = @ssl_keystore_password || report_invalid_config!('`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is configured')
|
142
|
+
elsif @ssl_keystore_password
|
143
|
+
report_invalid_config!('`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured')
|
144
|
+
else
|
145
|
+
report_invalid_config!('SSL identity MUST be configured with either `ssl_certificate`/`ssl_key` or `ssl_keystore_*`')
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
def ssl_trust_options
|
151
|
+
{
|
152
|
+
'ssl_client_authentication' => @ssl_client_authentication,
|
153
|
+
}.tap do |trust_options|
|
154
|
+
if @ssl_certificate_authorities&.any?
|
155
|
+
if @ssl_client_authentication == 'none'
|
156
|
+
report_invalid_config!('`ssl_certificate_authorities` is not supported because `ssl_client_authentication => none`')
|
157
|
+
end
|
158
|
+
|
159
|
+
trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def inner_json_lines_codec_options
|
165
|
+
@_inner_json_lines_codec_options ||= {
|
166
|
+
# enrichment avoidance
|
167
|
+
'ecs_compatibility' => 'disabled',
|
168
|
+
}
|
169
|
+
end
|
170
|
+
|
171
|
+
def report_invalid_config!(message)
|
172
|
+
fail(LogStash::ConfigurationError, message)
|
173
|
+
end
|
174
|
+
|
175
|
+
class QueueWrapper
|
176
|
+
def initialize(wrapped_queue)
|
177
|
+
@wrapped_queue = wrapped_queue
|
178
|
+
end
|
179
|
+
|
180
|
+
def << (event)
|
181
|
+
event.remove('[@metadata][void]')
|
182
|
+
@wrapped_queue << event
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'logstash/outputs/base'
|
4
|
+
require 'logstash/namespace'
|
5
|
+
|
6
|
+
require "logstash/plugin_mixins/plugin_factory_support"
|
7
|
+
|
8
|
+
class LogStash::Outputs::Logstash < LogStash::Outputs::Base
|
9
|
+
include LogStash::PluginMixins::PluginFactorySupport
|
10
|
+
|
11
|
+
config_name "logstash"
|
12
|
+
|
13
|
+
config :host, :validate => :string, :required => true
|
14
|
+
config :port, :validate => :number, :required => true
|
15
|
+
|
16
|
+
# optional username/password credentials
|
17
|
+
config :username, :validate => :string, :required => false
|
18
|
+
config :password, :validate => :password, :required => false
|
19
|
+
|
20
|
+
config :ssl_enabled, :validate => :boolean, :default => true
|
21
|
+
|
22
|
+
# SSL:IDENTITY:SOURCE cert/key pair
|
23
|
+
config :ssl_certificate, :validate => :path
|
24
|
+
config :ssl_key, :validate => :path
|
25
|
+
|
26
|
+
# SSL:IDENTITY:SOURCE keystore
|
27
|
+
config :ssl_keystore_path, :validate => :path
|
28
|
+
config :ssl_keystore_password, :validate => :password
|
29
|
+
|
30
|
+
# SSL:TRUST:CONFIG
|
31
|
+
config :ssl_verification_mode, :validate => %w(full none), :default => 'full'
|
32
|
+
|
33
|
+
# SSL:TRUST:SOURCE ca file
|
34
|
+
config :ssl_certificate_authorities, :validate => :path, :list => true
|
35
|
+
|
36
|
+
# SSL:TRUST:SOURCE truststore
|
37
|
+
config :ssl_truststore_path, :validate => :path
|
38
|
+
config :ssl_truststore_password, :validate => :password
|
39
|
+
|
40
|
+
# SSL:TUNING
|
41
|
+
config :ssl_supported_protocols, :validate => :string, :list => true
|
42
|
+
|
43
|
+
def initialize(*a)
|
44
|
+
super
|
45
|
+
|
46
|
+
if original_params.include?('codec')
|
47
|
+
fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
|
48
|
+
end
|
49
|
+
|
50
|
+
if @ssl_certificate_authorities && @ssl_certificate_authorities.size > 1
|
51
|
+
fail LogStash::ConfigurationError, 'The `logstash` output supports at most one `ssl_certificate_authorities` path'
|
52
|
+
end
|
53
|
+
|
54
|
+
logger.debug("initializing inner HTTP output plugin")
|
55
|
+
@internal_http = plugin_factory.output('http').new(inner_http_output_options)
|
56
|
+
logger.debug("inner HTTP output plugin has been initialized")
|
57
|
+
end
|
58
|
+
|
59
|
+
def register
|
60
|
+
logger.debug("registering inner HTTP output plugin")
|
61
|
+
@internal_http.register
|
62
|
+
logger.debug("inner HTTP output plugin has been registered")
|
63
|
+
end
|
64
|
+
|
65
|
+
def multi_receive(events)
|
66
|
+
return if events.empty?
|
67
|
+
logger.trace("proxying #{events.size} events to inner HTTP plugin")
|
68
|
+
@internal_http.multi_receive(events)
|
69
|
+
rescue => e
|
70
|
+
logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
|
71
|
+
raise
|
72
|
+
end
|
73
|
+
|
74
|
+
def stop
|
75
|
+
logger.debug("stopping inner HTTP output plugin")
|
76
|
+
@internal_http.stop
|
77
|
+
logger.debug('inner HTTP output plugin has been stopped')
|
78
|
+
end
|
79
|
+
|
80
|
+
def close
|
81
|
+
logger.debug("closing inner HTTP output plugin")
|
82
|
+
@internal_http.close
|
83
|
+
logger.debug('inner HTTP output plugin has been closed')
|
84
|
+
end
|
85
|
+
|
86
|
+
def inner_http_output_options
|
87
|
+
@_inner_http_output_options ||= begin
|
88
|
+
http_options = {
|
89
|
+
'url' => "#{@ssl_enabled ? 'https' : 'http'}://#{@host}:#{@port}",
|
90
|
+
'http_method' => 'post',
|
91
|
+
'retry_non_idempotent' => 'true',
|
92
|
+
|
93
|
+
# non-configurable codec
|
94
|
+
'content_type' => 'application/x-ndjson',
|
95
|
+
'format' => 'json_batch',
|
96
|
+
}
|
97
|
+
|
98
|
+
if @username
|
99
|
+
http_options['user'] = @username
|
100
|
+
http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
|
101
|
+
logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
|
102
|
+
elsif @password
|
103
|
+
fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
|
104
|
+
end
|
105
|
+
|
106
|
+
if @ssl_enabled == false
|
107
|
+
rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
|
108
|
+
fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
|
109
|
+
else
|
110
|
+
http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
|
111
|
+
|
112
|
+
http_options.merge!(ssl_identity_options)
|
113
|
+
http_options.merge!(ssl_trust_options)
|
114
|
+
end
|
115
|
+
|
116
|
+
http_options
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def ssl_identity_options
|
121
|
+
if @ssl_certificate && @ssl_keystore_path
|
122
|
+
fail(LogStash::ConfigurationError, 'SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
|
123
|
+
elsif @ssl_certificate
|
124
|
+
return {
|
125
|
+
'ssl_certificate' => @ssl_certificate,
|
126
|
+
'ssl_key' => @ssl_key || fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided"),
|
127
|
+
}
|
128
|
+
elsif @ssl_key
|
129
|
+
fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
|
130
|
+
elsif @ssl_keystore_path
|
131
|
+
return {
|
132
|
+
'ssl_keystore_path' => @ssl_keystore_path,
|
133
|
+
'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
|
134
|
+
}
|
135
|
+
elsif @ssl_keystore_password
|
136
|
+
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
|
137
|
+
else
|
138
|
+
return {}
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def ssl_trust_options
|
143
|
+
{
|
144
|
+
'ssl_verification_mode' => @ssl_verification_mode,
|
145
|
+
}.tap do |trust_options|
|
146
|
+
if @ssl_certificate_authorities&.any? && @ssl_truststore_path
|
147
|
+
fail(LogStash::ConfigurationError, 'SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both')
|
148
|
+
elsif @ssl_certificate_authorities&.any?
|
149
|
+
fail(LogStash::ConfigurationError, 'SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
|
150
|
+
|
151
|
+
trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities.first
|
152
|
+
elsif @ssl_truststore_path
|
153
|
+
fail(LogStash::ConfigurationError, 'SSL Truststore cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
|
154
|
+
|
155
|
+
trust_options['ssl_truststore_path'] = @ssl_truststore_path
|
156
|
+
trust_options['ssl_truststore_password'] = @ssl_truststore_password || fail(LogStash::ConfigurationError, '`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided')
|
157
|
+
elsif @ssl_truststore_password
|
158
|
+
fail(LogStash::ConfigurationError, '`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured')
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
INTEGRATION_LOGSTASH_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(INTEGRATION_LOGSTASH_VERSION)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "logstash-integration-logstash"
|
5
|
+
s.version = INTEGRATION_LOGSTASH_VERSION
|
6
|
+
s.licenses = ["Apache-2.0"]
|
7
|
+
s.summary = "Collection of Logstash plugins that enable sending events from one Logstash pipeline to another"
|
8
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
9
|
+
s.authors = ["Elastic"]
|
10
|
+
s.email = "info@elastic.co"
|
11
|
+
s.homepage = "https://www.elastic.co/logstash"
|
12
|
+
s.platform = "java"
|
13
|
+
s.metadata = {
|
14
|
+
"logstash_plugin" => "true",
|
15
|
+
"logstash_group" => "integration",
|
16
|
+
"integration_plugins" => %w(
|
17
|
+
logstash-input-logstash
|
18
|
+
logstash-output-logstash
|
19
|
+
).join(",")
|
20
|
+
}
|
21
|
+
|
22
|
+
s.require_paths = %w[lib vendor/jar-dependencies]
|
23
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "VERSION", "docs/**/*", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb"]
|
24
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
25
|
+
|
26
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
|
27
|
+
s.add_runtime_dependency "logstash-mixin-plugin_factory_support", "~> 1.0"
|
28
|
+
s.add_runtime_dependency "logstash-codec-json_lines", "~> 3.1"
|
29
|
+
|
30
|
+
s.add_runtime_dependency "logstash-input-http", ">= 3.7.0" # some params not available in older versions because they are renamed, such as `cacert` to `ssl_certificate_authorities`
|
31
|
+
s.add_runtime_dependency "logstash-output-http", ">= 5.6.0"
|
32
|
+
|
33
|
+
s.add_development_dependency "logstash-devutils"
|
34
|
+
s.add_development_dependency "rspec-collection_matchers"
|
35
|
+
s.add_development_dependency "random-port"
|
36
|
+
end
|