logstash-integration-logstash 0.0.1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +4 -0
  3. data/Gemfile +11 -0
  4. data/LICENSE +202 -0
  5. data/NOTICE.TXT +2 -0
  6. data/README.md +98 -0
  7. data/VERSION +1 -0
  8. data/docs/index.asciidoc +71 -0
  9. data/docs/input-logstash.asciidoc +252 -0
  10. data/docs/output-logstash.asciidoc +271 -0
  11. data/lib/logstash/inputs/logstash.rb +185 -0
  12. data/lib/logstash/outputs/logstash.rb +162 -0
  13. data/logstash-integration-logstash.gemspec +36 -0
  14. data/spec/fixtures/certs/generate.sh +69 -0
  15. data/spec/fixtures/certs/generated/README.txt +2 -0
  16. data/spec/fixtures/certs/generated/client_from_root.jks +0 -0
  17. data/spec/fixtures/certs/generated/client_from_root.key.pem +52 -0
  18. data/spec/fixtures/certs/generated/client_from_root.key.pkcs8.pem +54 -0
  19. data/spec/fixtures/certs/generated/client_from_root.p12 +0 -0
  20. data/spec/fixtures/certs/generated/client_from_root.pem +35 -0
  21. data/spec/fixtures/certs/generated/client_from_untrusted.jks +0 -0
  22. data/spec/fixtures/certs/generated/client_from_untrusted.key.pem +52 -0
  23. data/spec/fixtures/certs/generated/client_from_untrusted.key.pkcs8.pem +54 -0
  24. data/spec/fixtures/certs/generated/client_from_untrusted.p12 +0 -0
  25. data/spec/fixtures/certs/generated/client_from_untrusted.pem +35 -0
  26. data/spec/fixtures/certs/generated/client_self_signed.jks +0 -0
  27. data/spec/fixtures/certs/generated/client_self_signed.key.pem +52 -0
  28. data/spec/fixtures/certs/generated/client_self_signed.key.pkcs8.pem +54 -0
  29. data/spec/fixtures/certs/generated/client_self_signed.p12 +0 -0
  30. data/spec/fixtures/certs/generated/client_self_signed.pem +32 -0
  31. data/spec/fixtures/certs/generated/root.key.pem +52 -0
  32. data/spec/fixtures/certs/generated/root.pem +32 -0
  33. data/spec/fixtures/certs/generated/server_from_root-key-pkcs8.pem +52 -0
  34. data/spec/fixtures/certs/generated/server_from_root.jks +0 -0
  35. data/spec/fixtures/certs/generated/server_from_root.key.pem +52 -0
  36. data/spec/fixtures/certs/generated/server_from_root.key.pkcs8.pem +54 -0
  37. data/spec/fixtures/certs/generated/server_from_root.p12 +0 -0
  38. data/spec/fixtures/certs/generated/server_from_root.pem +37 -0
  39. data/spec/fixtures/certs/generated/untrusted.key.pem +52 -0
  40. data/spec/fixtures/certs/generated/untrusted.pem +32 -0
  41. data/spec/fixtures/certs/openssl.cnf +57 -0
  42. data/spec/spec_helper.rb +22 -0
  43. data/spec/unit/full_transmission_spec.rb +202 -0
  44. data/spec/unit/logstash_input_spec.rb +151 -0
  45. data/spec/unit/logstash_output_spec.rb +170 -0
  46. metadata +243 -0
@@ -0,0 +1,271 @@
1
+ :integration: logstash
2
+ :plugin: logstash
3
+ :type: output
4
+ :no_codec:
5
+
6
+ ///////////////////////////////////////////
7
+ START - GENERATED VARIABLES, DO NOT EDIT!
8
+ ///////////////////////////////////////////
9
+ :version: %VERSION%
10
+ :release_date: %RELEASE_DATE%
11
+ :changelog_url: %CHANGELOG_URL%
12
+ :include_path: ../../../../logstash/docs/include
13
+ ///////////////////////////////////////////
14
+ END - GENERATED VARIABLES, DO NOT EDIT!
15
+ ///////////////////////////////////////////
16
+
17
+ [id="plugins-{type}s-{plugin}"]
18
+
19
+ === Logstash output plugin
20
+
21
+ include::{include_path}/plugin_header-integration.asciidoc[]
22
+
23
+ ==== Description
24
+
25
+ Send events to a <<plugins-inputs-logstash>> in a pipeline that may be in another process or on another host.
26
+ You must have a TCP route to the port on an interface that the downstream input is bound to.
27
+
28
+ NOTE: Sending events to _any_ destination other than <<plugins-inputs-logstash>> is neither advised nor supported.
29
+ We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding.
30
+
31
+ [id="plugins-{type}s-{plugin}-minimum-config"]
32
+ ===== Minimum Configuration
33
+ [cols="2a,2a"]
34
+ |=======================================================================================================================
35
+ |SSL Enabled |SSL Disabled
36
+
37
+ |
38
+
39
+ [source]
40
+ ----
41
+ output {
42
+ logstash {
43
+ host => "10.0.0.123"
44
+ port => 8080
45
+ }
46
+ }
47
+ ----
48
+
49
+ |
50
+
51
+ [source]
52
+ ----
53
+ output {
54
+ logstash {
55
+ host => "10.0.0.123"
56
+ port => 8080
57
+ ssl_enabled
58
+ => false
59
+ }
60
+ }
61
+ ----
62
+
63
+ |=======================================================================================================================
64
+
65
+ [id="plugins-{type}s-{plugin}-config-connecting"]
66
+ ===== Configuration Concepts
67
+
68
+ This output plugin needs to be configured to connect to a <<plugins-inputs-logstash>> by specifying its <<plugins-{type}s-{plugin}-host>> and <<plugins-{type}s-{plugin}-port>>.
69
+ Depending on the downstream plugin's configuration, you may need to also configure SSL and/or credentials.
70
+
71
+ [id="plugins-{type}s-{plugin}-config-ssl-trust"]
72
+ ===== Security: SSL Trust
73
+
74
+ When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events.
75
+
76
+ It does so by ensuring that the server presents a currently-valid certificate with identity claims matching the <<plugins-{type}s-{plugin}-host>> we are configured to connect to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material.
77
+
78
+ The system trust store is used by default.
79
+ You can provide an _alternate_ source of trust with _ONE OF_:
80
+
81
+ * A PEM-formatted list of trusted certificate authorities (see <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>)
82
+ * A PKCS12- or JKS-formatted truststore (see <<plugins-{type}s-{plugin}-ssl_truststore_path>>)
83
+
84
+ [id="plugins-{type}s-{plugin}-config-ssl-identity"]
85
+ ===== Security: SSL Identity
86
+
87
+ If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_:
88
+
89
+ * JKS- or PKCS12-formatted Keystore (see <<plugins-{type}s-{plugin}-ssl_keystore_path>>)
90
+ * PKCS8-formatted Certificate/Key pair (see <<plugins-{type}s-{plugin}-ssl_certificate>>)
91
+
92
+ [id="plugins-{type}s-{plugin}-config-credentials"]
93
+ ===== Security: Credentials
94
+
95
+ If the downstream <<plugins-inputs-logstash>> is configured to require <<plugins-inputs-logstash-username>> and <<plugins-inputs-logstash-password>>,
96
+ you will need to configure this output with a matching <<plugins-{type}s-{plugin}-username>> and <<plugins-{type}s-{plugin}-password>>.
97
+
98
+ NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text.
99
+
100
+ [id="plugins-{type}s-{plugin}-options"]
101
+ ==== Logstash Output Configuration Options
102
+
103
+ This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
104
+
105
+ [cols="<,<,<",options="header",]
106
+ |=======================================================================
107
+ |Setting |Input type |Required
108
+ | <<plugins-{type}s-{plugin}-host>> |<<string,string>> |Yes
109
+ | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
110
+ | <<plugins-{type}s-{plugin}-port>> |<<number,number>> |Yes
111
+ | <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
112
+ | <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
113
+ | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
114
+ | <<plugins-{type}s-{plugin}-ssl_key>> | <<path,path>>|No
115
+ | <<plugins-{type}s-{plugin}-ssl_keystore_path>> | <<path,path>>|No
116
+ | <<plugins-{type}s-{plugin}-ssl_keystore_password>> | <<password,password>>|No
117
+ | <<plugins-{type}s-{plugin}-ssl_truststore_path>> | <<path,path>>|No
118
+ | <<plugins-{type}s-{plugin}-ssl_truststore_password>> | <<password,password>>|No
119
+ | <<plugins-{type}s-{plugin}-ssl_verification_mode>> | <<string,string>>, one of `["full", "none"]`|No
120
+ | <<plugins-{type}s-{plugin}-username>> |<<string,string>>|No
121
+ |=======================================================================
122
+
123
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
124
+ output plugins.
125
+
126
+ &nbsp;
127
+
128
+ [id="plugins-{type}s-{plugin}-host"]
129
+ ===== `host`
130
+
131
+ * Value type is a <<string,string>> ip address or hostname
132
+ * There is no default value
133
+
134
+ Specify which host to connect to by providing its ip address or resolvable hostname.
135
+
136
+ NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address.
137
+
138
+ [id="plugins-{type}s-{plugin}-port"]
139
+ ===== `port`
140
+
141
+ * Value type is a <<number,number>> port
142
+ * There is no default value
143
+
144
+ Specify which port to connect to.
145
+
146
+ [id="plugins-{type}s-{plugin}-password"]
147
+ ===== `password`
148
+
149
+ * Value type is <<password,password>>
150
+ * There is no default value for this setting.
151
+ * Required when <<plugins-{type}s-{plugin}-username>> is configured.
152
+
153
+ Password for password-based authentication.
154
+
155
+ When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
156
+
157
+ [id="plugins-{type}s-{plugin}-ssl_enabled"]
158
+ ===== `ssl_enabled`
159
+
160
+ * Value type is <<boolean,boolean>>
161
+ * Default value is `true`
162
+
163
+ Logstash-to-Logstash communication is secured by default.
164
+ When the downstream input plugin disables SSL, it must also be disabled here.
165
+
166
+ You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure.
167
+
168
+ [id="plugins-{type}s-{plugin}-ssl_certificate"]
169
+ ===== `ssl_certificate`
170
+
171
+ * Value type is <<path,path>>
172
+ * There is no default value for this setting.
173
+ * When present, <<plugins-{type}s-{plugin}-ssl_key>> is also required.
174
+ * Cannot be combined with configurations that disable SSL.
175
+
176
+ Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input.
177
+
178
+ [id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
179
+ ===== `ssl_certificate_authorities`
180
+
181
+ * Value type is a <<path,path>>
182
+ * There is no default value for this setting.
183
+ * Cannot be combined with configurations that disable SSL.
184
+ * Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
185
+
186
+ One or more PEM-encoded files defining certificate authorities for use in downstream input authentication.
187
+ This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input.
188
+
189
+ [id="plugins-{type}s-{plugin}-ssl_key"]
190
+ ===== `ssl_key`
191
+
192
+ * Value type is <<path,path>>
193
+ * There is no default value for this setting.
194
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_certificate>>
195
+ * Cannot be combined with configurations that disable SSL.
196
+
197
+ A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key.
198
+
199
+ [id="plugins-{type}s-{plugin}-ssl_keystore_path"]
200
+ ===== `ssl_keystore_path`
201
+
202
+ * Value type is <<path,path>>
203
+ * There is no default value for this setting.
204
+ * When present, <<plugins-{type}s-{plugin}-ssl_keystore_password>> is also required.
205
+ * Cannot be combined with configurations that disable SSL.
206
+
207
+ A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input.
208
+ The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
209
+
210
+ [id="plugins-{type}s-{plugin}-ssl_keystore_password"]
211
+ ===== `ssl_keystore_password`
212
+
213
+ * Value type is <<password,password>>
214
+ * There is no default value for this setting.
215
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_keystore_path>>
216
+ * Cannot be combined with configurations that disable SSL.
217
+
218
+ Password for the <<plugins-{type}s-{plugin}-ssl_keystore_path>>
219
+
220
+ [id="plugins-{type}s-{plugin}-ssl_truststore_path"]
221
+ ===== `ssl_truststore_path`
222
+
223
+ * Value type is <<path,path>>
224
+ * There is no default value for this setting.
225
+ * When present, <<plugins-{type}s-{plugin}-ssl_truststore_path>> is also required.
226
+ * Cannot be combined with configurations that disable SSL.
227
+ * Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
228
+
229
+ A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input.
230
+ The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
231
+
232
+ [id="plugins-{type}s-{plugin}-ssl_truststore_password"]
233
+ ===== `ssl_truststore_password`
234
+
235
+ * Value type is <<password,password>>
236
+ * There is no default value for this setting.
237
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_truststore_path>>
238
+ * Cannot be combined with configurations that disable SSL.
239
+
240
+ Password for the <<plugins-{type}s-{plugin}-ssl_truststore_path>>
241
+
242
+ [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
243
+ ===== `ssl_verification_mode`
244
+
245
+ * Value type is <<string,string>>
246
+ * The supported modes are:
247
+ ** `full`: verifies that a certificate provided by the client has an identity claim matching <<plugins-{type}s-{plugin}-host>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key.
248
+ ** `none`: performs no validation of the presented certificate
249
+
250
+ * The default value is `full`.
251
+ * Cannot be combined with configurations that disable SSL.
252
+
253
+ When communicating over SSL, this setting controls how the downstream input's certificate is verified.
254
+
255
+ [id="plugins-{type}s-{plugin}-username"]
256
+ ===== `username`
257
+
258
+ * Value type is <<string,string>>
259
+ * There is no default value for this setting.
260
+ * When present, <<plugins-{type}s-{plugin}-password>> is also required.
261
+
262
+ Username for password-based authentication.
263
+
264
+ When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
265
+
266
+ NOTE: when SSL is disabled, credentials will be transmitted in clear-text.
267
+
268
+ [id="plugins-{type}s-{plugin}-common-options"]
269
+ include::{include_path}/{type}.asciidoc[]
270
+
271
+ :default_codec!:
@@ -0,0 +1,185 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/inputs/base'
4
+ require 'logstash/namespace'
5
+
6
+ require "logstash/plugin_mixins/plugin_factory_support"
7
+
8
+ require 'logstash/codecs/json_lines'
9
+
10
+ class LogStash::Inputs::Logstash < LogStash::Inputs::Base
11
+ include LogStash::PluginMixins::PluginFactorySupport
12
+
13
+ config_name "logstash"
14
+
15
+ config :host, :validate => :string, :default => "0.0.0.0"
16
+ config :port, :validate => :number, :required => true
17
+
18
+ # optional username/password credentials
19
+ config :username, :validate => :string, :required => false
20
+ config :password, :validate => :password, :required => false
21
+
22
+ config :ssl_enabled, :validate => :boolean, :default => true
23
+
24
+ # SSL:IDENTITY:SOURCE cert/key pair
25
+ config :ssl_certificate, :validate => :path
26
+ config :ssl_key, :validate => :path
27
+ config :ssl_key_passphrase, :validate => :password
28
+
29
+ # SSL:IDENTITY:SOURCE keystore
30
+ config :ssl_keystore_path, :validate => :path
31
+ config :ssl_keystore_password, :validate => :password
32
+
33
+ # SSL:TRUST:CONFIG
34
+ config :ssl_client_authentication, :validate => %w(none optional required), :default => 'none'
35
+
36
+ # SSL:TRUST:SOURCE ca file
37
+ config :ssl_certificate_authorities, :validate => :path, :list => true
38
+
39
+ # SSL:TUNING
40
+ config :ssl_handshake_timeout, :validate => :number, default: 10_000
41
+ config :ssl_cipher_suites, :validate => :string, :list => true
42
+ config :ssl_supported_protocols, :validate => :string, :list => true
43
+
44
+ def initialize(*a)
45
+ super
46
+
47
+ if original_params.include?('codec')
48
+ report_invalid_config! 'The `logstash` input does not have an externally-configurable `codec`'
49
+ end
50
+
51
+ logger.debug("initializing inner HTTP input plugin")
52
+ @internal_http = plugin_factory.input('http').new(inner_http_input_options)
53
+ logger.debug("inner HTTP input plugin has been initialized")
54
+ end
55
+
56
+ def register
57
+ logger.debug("registering inner HTTP input plugin")
58
+ @internal_http.register
59
+ logger.debug("inner HTTP input plugin has been registered")
60
+ end
61
+
62
+ def run(queue)
63
+ logger.debug("starting inner HTTP input plugin")
64
+ @internal_http.run(QueueWrapper.new(queue))
65
+ logger.debug("inner HTTP input plugin has exited normally")
66
+ rescue => e
67
+ logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
68
+ raise
69
+ end
70
+
71
+ def stop
72
+ logger.debug("stopping inner HTTP input plugin")
73
+ @internal_http.stop
74
+ logger.debug('inner HTTP plugin has been stopped')
75
+ end
76
+
77
+ def close
78
+ logger.debug("closing inner HTTP input plugin")
79
+ @internal_http.close
80
+ logger.debug('inner HTTP plugin has been closed')
81
+ end
82
+
83
+ def inner_http_input_options
84
+ @_inner_http_input_options ||= begin
85
+ http_options = {
86
+ # directly-configurable
87
+ 'host' => @host,
88
+ 'port' => @port,
89
+
90
+ # non-configurable codec
91
+ 'codec' => plugin_factory.codec('json_lines').new(inner_json_lines_codec_options),
92
+ 'additional_codecs' => {},
93
+ 'response_headers' => { 'Accept' => 'application/x-ndjson' },
94
+
95
+ # enrichment avoidance
96
+ 'ecs_compatibility' => 'disabled',
97
+ 'remote_host_target_field' => '[@metadata][void]',
98
+ 'request_headers_target_field' => '[@metadata][void]',
99
+ }
100
+
101
+ if @username
102
+ http_options['user'] = @username
103
+ http_options['password'] = @password || report_invalid_config!('`password` is REQUIRED when `username` is provided')
104
+ logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
105
+ elsif @password
106
+ report_invalid_config!('`password` not allowed unless `username` is configured')
107
+ end
108
+
109
+ if @ssl_enabled == false
110
+ rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
111
+ report_invalid_config!("Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
112
+ else
113
+ http_options['ssl_enabled'] = true
114
+
115
+ http_options['ssl_cipher_suites'] = @ssl_cipher_suites if @original_params.include?('ssl_cipher_suites')
116
+ http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
117
+ http_options['ssl_handshake_timeout'] = @ssl_handshake_timeout
118
+
119
+ http_options.merge!(ssl_identity_options)
120
+ http_options.merge!(ssl_trust_options)
121
+ end
122
+
123
+ http_options
124
+ end
125
+ end
126
+
127
+ def ssl_identity_options
128
+ {}.tap do |identity_options|
129
+ if @ssl_certificate && @ssl_keystore_path
130
+ report_invalid_config!('SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
131
+ elsif @ssl_certificate
132
+ identity_options['ssl_certificate'] = @ssl_certificate
133
+ identity_options['ssl_key'] = @ssl_key || report_invalid_config!('`ssl_key` is required when `ssl_certificate` is configured')
134
+ identity_options['ssl_key_passphrase'] = @ssl_key_passphrase unless @ssl_key_passphrase.nil?
135
+ elsif @ssl_key
136
+ report_invalid_config!('`ssl_key` is not allowed unless `ssl_certificate` is configured')
137
+ elsif @ssl_key_passphrase
138
+ report_invalid_config!('`ssl_key_passphrase` is not allowed unless `ssl_key` is configured')
139
+ elsif @ssl_keystore_path
140
+ identity_options['ssl_keystore_path'] = @ssl_keystore_path
141
+ identity_options['ssl_keystore_password'] = @ssl_keystore_password || report_invalid_config!('`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is configured')
142
+ elsif @ssl_keystore_password
143
+ report_invalid_config!('`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured')
144
+ else
145
+ report_invalid_config!('SSL identity MUST be configured with either `ssl_certificate`/`ssl_key` or `ssl_keystore_*`')
146
+ end
147
+ end
148
+ end
149
+
150
+ def ssl_trust_options
151
+ {
152
+ 'ssl_client_authentication' => @ssl_client_authentication,
153
+ }.tap do |trust_options|
154
+ if @ssl_certificate_authorities&.any?
155
+ if @ssl_client_authentication == 'none'
156
+ report_invalid_config!('`ssl_certificate_authorities` is not supported because `ssl_client_authentication => none`')
157
+ end
158
+
159
+ trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities
160
+ end
161
+ end
162
+ end
163
+
164
+ def inner_json_lines_codec_options
165
+ @_inner_json_lines_codec_options ||= {
166
+ # enrichment avoidance
167
+ 'ecs_compatibility' => 'disabled',
168
+ }
169
+ end
170
+
171
+ def report_invalid_config!(message)
172
+ fail(LogStash::ConfigurationError, message)
173
+ end
174
+
175
+ class QueueWrapper
176
+ def initialize(wrapped_queue)
177
+ @wrapped_queue = wrapped_queue
178
+ end
179
+
180
+ def << (event)
181
+ event.remove('[@metadata][void]')
182
+ @wrapped_queue << event
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,162 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/outputs/base'
4
+ require 'logstash/namespace'
5
+
6
+ require "logstash/plugin_mixins/plugin_factory_support"
7
+
8
+ class LogStash::Outputs::Logstash < LogStash::Outputs::Base
9
+ include LogStash::PluginMixins::PluginFactorySupport
10
+
11
+ config_name "logstash"
12
+
13
+ config :host, :validate => :string, :required => true
14
+ config :port, :validate => :number, :required => true
15
+
16
+ # optional username/password credentials
17
+ config :username, :validate => :string, :required => false
18
+ config :password, :validate => :password, :required => false
19
+
20
+ config :ssl_enabled, :validate => :boolean, :default => true
21
+
22
+ # SSL:IDENTITY:SOURCE cert/key pair
23
+ config :ssl_certificate, :validate => :path
24
+ config :ssl_key, :validate => :path
25
+
26
+ # SSL:IDENTITY:SOURCE keystore
27
+ config :ssl_keystore_path, :validate => :path
28
+ config :ssl_keystore_password, :validate => :password
29
+
30
+ # SSL:TRUST:CONFIG
31
+ config :ssl_verification_mode, :validate => %w(full none), :default => 'full'
32
+
33
+ # SSL:TRUST:SOURCE ca file
34
+ config :ssl_certificate_authorities, :validate => :path, :list => true
35
+
36
+ # SSL:TRUST:SOURCE truststore
37
+ config :ssl_truststore_path, :validate => :path
38
+ config :ssl_truststore_password, :validate => :password
39
+
40
+ # SSL:TUNING
41
+ config :ssl_supported_protocols, :validate => :string, :list => true
42
+
43
+ def initialize(*a)
44
+ super
45
+
46
+ if original_params.include?('codec')
47
+ fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
48
+ end
49
+
50
+ if @ssl_certificate_authorities && @ssl_certificate_authorities.size > 1
51
+ fail LogStash::ConfigurationError, 'The `logstash` output supports at most one `ssl_certificate_authorities` path'
52
+ end
53
+
54
+ logger.debug("initializing inner HTTP output plugin")
55
+ @internal_http = plugin_factory.output('http').new(inner_http_output_options)
56
+ logger.debug("inner HTTP output plugin has been initialized")
57
+ end
58
+
59
+ def register
60
+ logger.debug("registering inner HTTP output plugin")
61
+ @internal_http.register
62
+ logger.debug("inner HTTP output plugin has been registered")
63
+ end
64
+
65
+ def multi_receive(events)
66
+ return if events.empty?
67
+ logger.trace("proxying #{events.size} events to inner HTTP plugin")
68
+ @internal_http.multi_receive(events)
69
+ rescue => e
70
+ logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
71
+ raise
72
+ end
73
+
74
+ def stop
75
+ logger.debug("stopping inner HTTP output plugin")
76
+ @internal_http.stop
77
+ logger.debug('inner HTTP output plugin has been stopped')
78
+ end
79
+
80
+ def close
81
+ logger.debug("closing inner HTTP output plugin")
82
+ @internal_http.close
83
+ logger.debug('inner HTTP output plugin has been closed')
84
+ end
85
+
86
+ def inner_http_output_options
87
+ @_inner_http_output_options ||= begin
88
+ http_options = {
89
+ 'url' => "#{@ssl_enabled ? 'https' : 'http'}://#{@host}:#{@port}",
90
+ 'http_method' => 'post',
91
+ 'retry_non_idempotent' => 'true',
92
+
93
+ # non-configurable codec
94
+ 'content_type' => 'application/x-ndjson',
95
+ 'format' => 'json_batch',
96
+ }
97
+
98
+ if @username
99
+ http_options['user'] = @username
100
+ http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
101
+ logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
102
+ elsif @password
103
+ fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
104
+ end
105
+
106
+ if @ssl_enabled == false
107
+ rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
108
+ fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
109
+ else
110
+ http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
111
+
112
+ http_options.merge!(ssl_identity_options)
113
+ http_options.merge!(ssl_trust_options)
114
+ end
115
+
116
+ http_options
117
+ end
118
+ end
119
+
120
+ def ssl_identity_options
121
+ if @ssl_certificate && @ssl_keystore_path
122
+ fail(LogStash::ConfigurationError, 'SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
123
+ elsif @ssl_certificate
124
+ return {
125
+ 'ssl_certificate' => @ssl_certificate,
126
+ 'ssl_key' => @ssl_key || fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided"),
127
+ }
128
+ elsif @ssl_key
129
+ fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
130
+ elsif @ssl_keystore_path
131
+ return {
132
+ 'ssl_keystore_path' => @ssl_keystore_path,
133
+ 'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
134
+ }
135
+ elsif @ssl_keystore_password
136
+ fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
137
+ else
138
+ return {}
139
+ end
140
+ end
141
+
142
+ def ssl_trust_options
143
+ {
144
+ 'ssl_verification_mode' => @ssl_verification_mode,
145
+ }.tap do |trust_options|
146
+ if @ssl_certificate_authorities&.any? && @ssl_truststore_path
147
+ fail(LogStash::ConfigurationError, 'SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both')
148
+ elsif @ssl_certificate_authorities&.any?
149
+ fail(LogStash::ConfigurationError, 'SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
150
+
151
+ trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities.first
152
+ elsif @ssl_truststore_path
153
+ fail(LogStash::ConfigurationError, 'SSL Truststore cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
154
+
155
+ trust_options['ssl_truststore_path'] = @ssl_truststore_path
156
+ trust_options['ssl_truststore_password'] = @ssl_truststore_password || fail(LogStash::ConfigurationError, '`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided')
157
+ elsif @ssl_truststore_password
158
+ fail(LogStash::ConfigurationError, '`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured')
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,36 @@
1
+ INTEGRATION_LOGSTASH_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(INTEGRATION_LOGSTASH_VERSION)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "logstash-integration-logstash"
5
+ s.version = INTEGRATION_LOGSTASH_VERSION
6
+ s.licenses = ["Apache-2.0"]
7
+ s.summary = "Collection of Logstash plugins that enable sending events from one Logstash pipeline to another"
8
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
9
+ s.authors = ["Elastic"]
10
+ s.email = "info@elastic.co"
11
+ s.homepage = "https://www.elastic.co/logstash"
12
+ s.platform = "java"
13
+ s.metadata = {
14
+ "logstash_plugin" => "true",
15
+ "logstash_group" => "integration",
16
+ "integration_plugins" => %w(
17
+ logstash-input-logstash
18
+ logstash-output-logstash
19
+ ).join(",")
20
+ }
21
+
22
+ s.require_paths = %w[lib vendor/jar-dependencies]
23
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "VERSION", "docs/**/*", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb"]
24
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
25
+
26
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
27
+ s.add_runtime_dependency "logstash-mixin-plugin_factory_support", "~> 1.0"
28
+ s.add_runtime_dependency "logstash-codec-json_lines", "~> 3.1"
29
+
30
+ s.add_runtime_dependency "logstash-input-http", ">= 3.7.0" # some params not available in older versions because they are renamed, such as `cacert` to `ssl_certificate_authorities`
31
+ s.add_runtime_dependency "logstash-output-http", ">= 5.6.0"
32
+
33
+ s.add_development_dependency "logstash-devutils"
34
+ s.add_development_dependency "rspec-collection_matchers"
35
+ s.add_development_dependency "random-port"
36
+ end