logstash-integration-logstash 0.0.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +4 -0
  3. data/Gemfile +11 -0
  4. data/LICENSE +202 -0
  5. data/NOTICE.TXT +2 -0
  6. data/README.md +98 -0
  7. data/VERSION +1 -0
  8. data/docs/index.asciidoc +71 -0
  9. data/docs/input-logstash.asciidoc +252 -0
  10. data/docs/output-logstash.asciidoc +271 -0
  11. data/lib/logstash/inputs/logstash.rb +185 -0
  12. data/lib/logstash/outputs/logstash.rb +162 -0
  13. data/logstash-integration-logstash.gemspec +36 -0
  14. data/spec/fixtures/certs/generate.sh +69 -0
  15. data/spec/fixtures/certs/generated/README.txt +2 -0
  16. data/spec/fixtures/certs/generated/client_from_root.jks +0 -0
  17. data/spec/fixtures/certs/generated/client_from_root.key.pem +52 -0
  18. data/spec/fixtures/certs/generated/client_from_root.key.pkcs8.pem +54 -0
  19. data/spec/fixtures/certs/generated/client_from_root.p12 +0 -0
  20. data/spec/fixtures/certs/generated/client_from_root.pem +35 -0
  21. data/spec/fixtures/certs/generated/client_from_untrusted.jks +0 -0
  22. data/spec/fixtures/certs/generated/client_from_untrusted.key.pem +52 -0
  23. data/spec/fixtures/certs/generated/client_from_untrusted.key.pkcs8.pem +54 -0
  24. data/spec/fixtures/certs/generated/client_from_untrusted.p12 +0 -0
  25. data/spec/fixtures/certs/generated/client_from_untrusted.pem +35 -0
  26. data/spec/fixtures/certs/generated/client_self_signed.jks +0 -0
  27. data/spec/fixtures/certs/generated/client_self_signed.key.pem +52 -0
  28. data/spec/fixtures/certs/generated/client_self_signed.key.pkcs8.pem +54 -0
  29. data/spec/fixtures/certs/generated/client_self_signed.p12 +0 -0
  30. data/spec/fixtures/certs/generated/client_self_signed.pem +32 -0
  31. data/spec/fixtures/certs/generated/root.key.pem +52 -0
  32. data/spec/fixtures/certs/generated/root.pem +32 -0
  33. data/spec/fixtures/certs/generated/server_from_root-key-pkcs8.pem +52 -0
  34. data/spec/fixtures/certs/generated/server_from_root.jks +0 -0
  35. data/spec/fixtures/certs/generated/server_from_root.key.pem +52 -0
  36. data/spec/fixtures/certs/generated/server_from_root.key.pkcs8.pem +54 -0
  37. data/spec/fixtures/certs/generated/server_from_root.p12 +0 -0
  38. data/spec/fixtures/certs/generated/server_from_root.pem +37 -0
  39. data/spec/fixtures/certs/generated/untrusted.key.pem +52 -0
  40. data/spec/fixtures/certs/generated/untrusted.pem +32 -0
  41. data/spec/fixtures/certs/openssl.cnf +57 -0
  42. data/spec/spec_helper.rb +22 -0
  43. data/spec/unit/full_transmission_spec.rb +202 -0
  44. data/spec/unit/logstash_input_spec.rb +151 -0
  45. data/spec/unit/logstash_output_spec.rb +170 -0
  46. metadata +243 -0
@@ -0,0 +1,271 @@
1
+ :integration: logstash
2
+ :plugin: logstash
3
+ :type: output
4
+ :no_codec:
5
+
6
+ ///////////////////////////////////////////
7
+ START - GENERATED VARIABLES, DO NOT EDIT!
8
+ ///////////////////////////////////////////
9
+ :version: %VERSION%
10
+ :release_date: %RELEASE_DATE%
11
+ :changelog_url: %CHANGELOG_URL%
12
+ :include_path: ../../../../logstash/docs/include
13
+ ///////////////////////////////////////////
14
+ END - GENERATED VARIABLES, DO NOT EDIT!
15
+ ///////////////////////////////////////////
16
+
17
+ [id="plugins-{type}s-{plugin}"]
18
+
19
+ === Logstash output plugin
20
+
21
+ include::{include_path}/plugin_header-integration.asciidoc[]
22
+
23
+ ==== Description
24
+
25
+ Send events to a <<plugins-inputs-logstash>> in a pipeline that may be in another process or on another host.
26
+ You must have a TCP route to the port on an interface that the downstream input is bound to.
27
+
28
+ NOTE: Sending events to _any_ destination other than <<plugins-inputs-logstash>> is neither advised nor supported.
29
+ We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding.
30
+
31
+ [id="plugins-{type}s-{plugin}-minimum-config"]
32
+ ===== Minimum Configuration
33
+ [cols="2a,2a"]
34
+ |=======================================================================================================================
35
+ |SSL Enabled |SSL Disabled
36
+
37
+ |
38
+
39
+ [source]
40
+ ----
41
+ output {
42
+ logstash {
43
+ host => "10.0.0.123"
44
+ port => 8080
45
+ }
46
+ }
47
+ ----
48
+
49
+ |
50
+
51
+ [source]
52
+ ----
53
+ output {
54
+ logstash {
55
+ host => "10.0.0.123"
56
+ port => 8080
57
+ ssl_enabled
58
+ => false
59
+ }
60
+ }
61
+ ----
62
+
63
+ |=======================================================================================================================
64
+
65
+ [id="plugins-{type}s-{plugin}-config-connecting"]
66
+ ===== Configuration Concepts
67
+
68
+ This output plugin needs to be configured to connect to a <<plugins-inputs-logstash>> by specifying its <<plugins-{type}s-{plugin}-host>> and <<plugins-{type}s-{plugin}-port>>.
69
+ Depending on the downstream plugin's configuration, you may need to also configure SSL and/or credentials.
70
+
71
+ [id="plugins-{type}s-{plugin}-config-ssl-trust"]
72
+ ===== Security: SSL Trust
73
+
74
+ When communicating over SSL, this plugin establishes trust of the server it connects to before transmitting credentials or events.
75
+
76
+ It does so by ensuring that the server presents a currently-valid certificate with identity claims matching the <<plugins-{type}s-{plugin}-host>> we are configured to connect to, signed by a trusted signing authority, along with proof-of-possession of the associated private key material.
77
+
78
+ The system trust store is used by default.
79
+ You can provide an _alternate_ source of trust with _ONE OF_:
80
+
81
+ * A PEM-formatted list of trusted certificate authorities (see <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>)
82
+ * A PKCS12- or JKS-formatted truststore (see <<plugins-{type}s-{plugin}-ssl_truststore_path>>)
83
+
84
+ [id="plugins-{type}s-{plugin}-config-ssl-identity"]
85
+ ===== Security: SSL Identity
86
+
87
+ If the downstream input plugin is configured to request or require client authentication, you can configure this plugin to provide its proof-of-identity with _ONE OF_:
88
+
89
+ * JKS- or PKCS12-formatted Keystore (see <<plugins-{type}s-{plugin}-ssl_keystore_path>>)
90
+ * PKCS8-formatted Certificate/Key pair (see <<plugins-{type}s-{plugin}-ssl_certificate>>)
91
+
92
+ [id="plugins-{type}s-{plugin}-config-credentials"]
93
+ ===== Security: Credentials
94
+
95
+ If the downstream <<plugins-inputs-logstash>> is configured to require <<plugins-inputs-logstash-username>> and <<plugins-inputs-logstash-password>>,
96
+ you will need to configure this output with a matching <<plugins-{type}s-{plugin}-username>> and <<plugins-{type}s-{plugin}-password>>.
97
+
98
+ NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text.
99
+
100
+ [id="plugins-{type}s-{plugin}-options"]
101
+ ==== Logstash Output Configuration Options
102
+
103
+ This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
104
+
105
+ [cols="<,<,<",options="header",]
106
+ |=======================================================================
107
+ |Setting |Input type |Required
108
+ | <<plugins-{type}s-{plugin}-host>> |<<string,string>> |Yes
109
+ | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
110
+ | <<plugins-{type}s-{plugin}-port>> |<<number,number>> |Yes
111
+ | <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
112
+ | <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
113
+ | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
114
+ | <<plugins-{type}s-{plugin}-ssl_key>> | <<path,path>>|No
115
+ | <<plugins-{type}s-{plugin}-ssl_keystore_path>> | <<path,path>>|No
116
+ | <<plugins-{type}s-{plugin}-ssl_keystore_password>> | <<password,password>>|No
117
+ | <<plugins-{type}s-{plugin}-ssl_truststore_path>> | <<path,path>>|No
118
+ | <<plugins-{type}s-{plugin}-ssl_truststore_password>> | <<password,password>>|No
119
+ | <<plugins-{type}s-{plugin}-ssl_verification_mode>> | <<string,string>>, one of `["full", "none"]`|No
120
+ | <<plugins-{type}s-{plugin}-username>> |<<string,string>>|No
121
+ |=======================================================================
122
+
123
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
124
+ output plugins.
125
+
126
+ &nbsp;
127
+
128
+ [id="plugins-{type}s-{plugin}-host"]
129
+ ===== `host`
130
+
131
+ * Value type is a <<string,string>> ip address or hostname
132
+ * There is no default value
133
+
134
+ Specify which host to connect to by providing its ip address or resolvable hostname.
135
+
136
+ NOTE: when using SSL, the server that responds must present a certificated with identity claim matching this host name or ip address.
137
+
138
+ [id="plugins-{type}s-{plugin}-port"]
139
+ ===== `port`
140
+
141
+ * Value type is a <<number,number>> port
142
+ * There is no default value
143
+
144
+ Specify which port to connect to.
145
+
146
+ [id="plugins-{type}s-{plugin}-password"]
147
+ ===== `password`
148
+
149
+ * Value type is <<password,password>>
150
+ * There is no default value for this setting.
151
+ * Required when <<plugins-{type}s-{plugin}-username>> is configured.
152
+
153
+ Password for password-based authentication.
154
+
155
+ When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
156
+
157
+ [id="plugins-{type}s-{plugin}-ssl_enabled"]
158
+ ===== `ssl_enabled`
159
+
160
+ * Value type is <<boolean,boolean>>
161
+ * Default value is `true`
162
+
163
+ Logstash-to-Logstash communication is secured by default.
164
+ When the downstream input plugin disables SSL, it must also be disabled here.
165
+
166
+ You can disable SSL with `+ssl_enabled => false+`. When disabled, setting any `ssl_*` configuration causes configuration failure.
167
+
168
+ [id="plugins-{type}s-{plugin}-ssl_certificate"]
169
+ ===== `ssl_certificate`
170
+
171
+ * Value type is <<path,path>>
172
+ * There is no default value for this setting.
173
+ * When present, <<plugins-{type}s-{plugin}-ssl_key>> is also required.
174
+ * Cannot be combined with configurations that disable SSL.
175
+
176
+ Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to connecting downstream input.
177
+
178
+ [id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
179
+ ===== `ssl_certificate_authorities`
180
+
181
+ * Value type is a <<path,path>>
182
+ * There is no default value for this setting.
183
+ * Cannot be combined with configurations that disable SSL.
184
+ * Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
185
+
186
+ One or more PEM-encoded files defining certificate authorities for use in downstream input authentication.
187
+ This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by downstream input.
188
+
189
+ [id="plugins-{type}s-{plugin}-ssl_key"]
190
+ ===== `ssl_key`
191
+
192
+ * Value type is <<path,path>>
193
+ * There is no default value for this setting.
194
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_certificate>>
195
+ * Cannot be combined with configurations that disable SSL.
196
+
197
+ A path to an PEM-encoded _unencrypted_ PKCS8 SSL certificate key.
198
+
199
+ [id="plugins-{type}s-{plugin}-ssl_keystore_path"]
200
+ ===== `ssl_keystore_path`
201
+
202
+ * Value type is <<path,path>>
203
+ * There is no default value for this setting.
204
+ * When present, <<plugins-{type}s-{plugin}-ssl_keystore_password>> is also required.
205
+ * Cannot be combined with configurations that disable SSL.
206
+
207
+ A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to the downstream input.
208
+ The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
209
+
210
+ [id="plugins-{type}s-{plugin}-ssl_keystore_password"]
211
+ ===== `ssl_keystore_password`
212
+
213
+ * Value type is <<password,password>>
214
+ * There is no default value for this setting.
215
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_keystore_path>>
216
+ * Cannot be combined with configurations that disable SSL.
217
+
218
+ Password for the <<plugins-{type}s-{plugin}-ssl_keystore_path>>
219
+
220
+ [id="plugins-{type}s-{plugin}-ssl_truststore_path"]
221
+ ===== `ssl_truststore_path`
222
+
223
+ * Value type is <<path,path>>
224
+ * There is no default value for this setting.
225
+ * When present, <<plugins-{type}s-{plugin}-ssl_truststore_path>> is also required.
226
+ * Cannot be combined with configurations that disable SSL.
227
+ * Cannot be combined with <<plugins-{type}s-{plugin}-ssl_verification_mode, `+ssl_verification_mode => none+`>>.
228
+
229
+ A path to a JKS- or PKCS12-formatted truststore with which to validate the identity claims of the downstream input.
230
+ The provided identity will be used if the downstream input enables <<plugins-{type}s-{plugin}-config-ssl-trust,SSL client authentication>>.
231
+
232
+ [id="plugins-{type}s-{plugin}-ssl_truststore_password"]
233
+ ===== `ssl_truststore_password`
234
+
235
+ * Value type is <<password,password>>
236
+ * There is no default value for this setting.
237
+ * Required when connection identity is configured with <<plugins-{type}s-{plugin}-ssl_truststore_path>>
238
+ * Cannot be combined with configurations that disable SSL.
239
+
240
+ Password for the <<plugins-{type}s-{plugin}-ssl_truststore_path>>
241
+
242
+ [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
243
+ ===== `ssl_verification_mode`
244
+
245
+ * Value type is <<string,string>>
246
+ * The supported modes are:
247
+ ** `full`: verifies that a certificate provided by the client has an identity claim matching <<plugins-{type}s-{plugin}-host>>, is signed by a trusted authority (CA), is within its valid date range, and that the client has possession of the associated key.
248
+ ** `none`: performs no validation of the presented certificate
249
+
250
+ * The default value is `full`.
251
+ * Cannot be combined with configurations that disable SSL.
252
+
253
+ When communicating over SSL, this setting controls how the downstream input's certificate is verified.
254
+
255
+ [id="plugins-{type}s-{plugin}-username"]
256
+ ===== `username`
257
+
258
+ * Value type is <<string,string>>
259
+ * There is no default value for this setting.
260
+ * When present, <<plugins-{type}s-{plugin}-password>> is also required.
261
+
262
+ Username for password-based authentication.
263
+
264
+ When the downstream input plugin is configured with a `username` and `password`, you must also configure upstream outputs with a matching `username`/`password` pair.
265
+
266
+ NOTE: when SSL is disabled, credentials will be transmitted in clear-text.
267
+
268
+ [id="plugins-{type}s-{plugin}-common-options"]
269
+ include::{include_path}/{type}.asciidoc[]
270
+
271
+ :default_codec!:
@@ -0,0 +1,185 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/inputs/base'
4
+ require 'logstash/namespace'
5
+
6
+ require "logstash/plugin_mixins/plugin_factory_support"
7
+
8
+ require 'logstash/codecs/json_lines'
9
+
10
+ class LogStash::Inputs::Logstash < LogStash::Inputs::Base
11
+ include LogStash::PluginMixins::PluginFactorySupport
12
+
13
+ config_name "logstash"
14
+
15
+ config :host, :validate => :string, :default => "0.0.0.0"
16
+ config :port, :validate => :number, :required => true
17
+
18
+ # optional username/password credentials
19
+ config :username, :validate => :string, :required => false
20
+ config :password, :validate => :password, :required => false
21
+
22
+ config :ssl_enabled, :validate => :boolean, :default => true
23
+
24
+ # SSL:IDENTITY:SOURCE cert/key pair
25
+ config :ssl_certificate, :validate => :path
26
+ config :ssl_key, :validate => :path
27
+ config :ssl_key_passphrase, :validate => :password
28
+
29
+ # SSL:IDENTITY:SOURCE keystore
30
+ config :ssl_keystore_path, :validate => :path
31
+ config :ssl_keystore_password, :validate => :password
32
+
33
+ # SSL:TRUST:CONFIG
34
+ config :ssl_client_authentication, :validate => %w(none optional required), :default => 'none'
35
+
36
+ # SSL:TRUST:SOURCE ca file
37
+ config :ssl_certificate_authorities, :validate => :path, :list => true
38
+
39
+ # SSL:TUNING
40
+ config :ssl_handshake_timeout, :validate => :number, default: 10_000
41
+ config :ssl_cipher_suites, :validate => :string, :list => true
42
+ config :ssl_supported_protocols, :validate => :string, :list => true
43
+
44
+ def initialize(*a)
45
+ super
46
+
47
+ if original_params.include?('codec')
48
+ report_invalid_config! 'The `logstash` input does not have an externally-configurable `codec`'
49
+ end
50
+
51
+ logger.debug("initializing inner HTTP input plugin")
52
+ @internal_http = plugin_factory.input('http').new(inner_http_input_options)
53
+ logger.debug("inner HTTP input plugin has been initialized")
54
+ end
55
+
56
+ def register
57
+ logger.debug("registering inner HTTP input plugin")
58
+ @internal_http.register
59
+ logger.debug("inner HTTP input plugin has been registered")
60
+ end
61
+
62
+ def run(queue)
63
+ logger.debug("starting inner HTTP input plugin")
64
+ @internal_http.run(QueueWrapper.new(queue))
65
+ logger.debug("inner HTTP input plugin has exited normally")
66
+ rescue => e
67
+ logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
68
+ raise
69
+ end
70
+
71
+ def stop
72
+ logger.debug("stopping inner HTTP input plugin")
73
+ @internal_http.stop
74
+ logger.debug('inner HTTP plugin has been stopped')
75
+ end
76
+
77
+ def close
78
+ logger.debug("closing inner HTTP input plugin")
79
+ @internal_http.close
80
+ logger.debug('inner HTTP plugin has been closed')
81
+ end
82
+
83
+ def inner_http_input_options
84
+ @_inner_http_input_options ||= begin
85
+ http_options = {
86
+ # directly-configurable
87
+ 'host' => @host,
88
+ 'port' => @port,
89
+
90
+ # non-configurable codec
91
+ 'codec' => plugin_factory.codec('json_lines').new(inner_json_lines_codec_options),
92
+ 'additional_codecs' => {},
93
+ 'response_headers' => { 'Accept' => 'application/x-ndjson' },
94
+
95
+ # enrichment avoidance
96
+ 'ecs_compatibility' => 'disabled',
97
+ 'remote_host_target_field' => '[@metadata][void]',
98
+ 'request_headers_target_field' => '[@metadata][void]',
99
+ }
100
+
101
+ if @username
102
+ http_options['user'] = @username
103
+ http_options['password'] = @password || report_invalid_config!('`password` is REQUIRED when `username` is provided')
104
+ logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
105
+ elsif @password
106
+ report_invalid_config!('`password` not allowed unless `username` is configured')
107
+ end
108
+
109
+ if @ssl_enabled == false
110
+ rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
111
+ report_invalid_config!("Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
112
+ else
113
+ http_options['ssl_enabled'] = true
114
+
115
+ http_options['ssl_cipher_suites'] = @ssl_cipher_suites if @original_params.include?('ssl_cipher_suites')
116
+ http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
117
+ http_options['ssl_handshake_timeout'] = @ssl_handshake_timeout
118
+
119
+ http_options.merge!(ssl_identity_options)
120
+ http_options.merge!(ssl_trust_options)
121
+ end
122
+
123
+ http_options
124
+ end
125
+ end
126
+
127
+ def ssl_identity_options
128
+ {}.tap do |identity_options|
129
+ if @ssl_certificate && @ssl_keystore_path
130
+ report_invalid_config!('SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
131
+ elsif @ssl_certificate
132
+ identity_options['ssl_certificate'] = @ssl_certificate
133
+ identity_options['ssl_key'] = @ssl_key || report_invalid_config!('`ssl_key` is required when `ssl_certificate` is configured')
134
+ identity_options['ssl_key_passphrase'] = @ssl_key_passphrase unless @ssl_key_passphrase.nil?
135
+ elsif @ssl_key
136
+ report_invalid_config!('`ssl_key` is not allowed unless `ssl_certificate` is configured')
137
+ elsif @ssl_key_passphrase
138
+ report_invalid_config!('`ssl_key_passphrase` is not allowed unless `ssl_key` is configured')
139
+ elsif @ssl_keystore_path
140
+ identity_options['ssl_keystore_path'] = @ssl_keystore_path
141
+ identity_options['ssl_keystore_password'] = @ssl_keystore_password || report_invalid_config!('`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is configured')
142
+ elsif @ssl_keystore_password
143
+ report_invalid_config!('`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured')
144
+ else
145
+ report_invalid_config!('SSL identity MUST be configured with either `ssl_certificate`/`ssl_key` or `ssl_keystore_*`')
146
+ end
147
+ end
148
+ end
149
+
150
+ def ssl_trust_options
151
+ {
152
+ 'ssl_client_authentication' => @ssl_client_authentication,
153
+ }.tap do |trust_options|
154
+ if @ssl_certificate_authorities&.any?
155
+ if @ssl_client_authentication == 'none'
156
+ report_invalid_config!('`ssl_certificate_authorities` is not supported because `ssl_client_authentication => none`')
157
+ end
158
+
159
+ trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities
160
+ end
161
+ end
162
+ end
163
+
164
+ def inner_json_lines_codec_options
165
+ @_inner_json_lines_codec_options ||= {
166
+ # enrichment avoidance
167
+ 'ecs_compatibility' => 'disabled',
168
+ }
169
+ end
170
+
171
+ def report_invalid_config!(message)
172
+ fail(LogStash::ConfigurationError, message)
173
+ end
174
+
175
+ class QueueWrapper
176
+ def initialize(wrapped_queue)
177
+ @wrapped_queue = wrapped_queue
178
+ end
179
+
180
+ def << (event)
181
+ event.remove('[@metadata][void]')
182
+ @wrapped_queue << event
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,162 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/outputs/base'
4
+ require 'logstash/namespace'
5
+
6
+ require "logstash/plugin_mixins/plugin_factory_support"
7
+
8
+ class LogStash::Outputs::Logstash < LogStash::Outputs::Base
9
+ include LogStash::PluginMixins::PluginFactorySupport
10
+
11
+ config_name "logstash"
12
+
13
+ config :host, :validate => :string, :required => true
14
+ config :port, :validate => :number, :required => true
15
+
16
+ # optional username/password credentials
17
+ config :username, :validate => :string, :required => false
18
+ config :password, :validate => :password, :required => false
19
+
20
+ config :ssl_enabled, :validate => :boolean, :default => true
21
+
22
+ # SSL:IDENTITY:SOURCE cert/key pair
23
+ config :ssl_certificate, :validate => :path
24
+ config :ssl_key, :validate => :path
25
+
26
+ # SSL:IDENTITY:SOURCE keystore
27
+ config :ssl_keystore_path, :validate => :path
28
+ config :ssl_keystore_password, :validate => :password
29
+
30
+ # SSL:TRUST:CONFIG
31
+ config :ssl_verification_mode, :validate => %w(full none), :default => 'full'
32
+
33
+ # SSL:TRUST:SOURCE ca file
34
+ config :ssl_certificate_authorities, :validate => :path, :list => true
35
+
36
+ # SSL:TRUST:SOURCE truststore
37
+ config :ssl_truststore_path, :validate => :path
38
+ config :ssl_truststore_password, :validate => :password
39
+
40
+ # SSL:TUNING
41
+ config :ssl_supported_protocols, :validate => :string, :list => true
42
+
43
+ def initialize(*a)
44
+ super
45
+
46
+ if original_params.include?('codec')
47
+ fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
48
+ end
49
+
50
+ if @ssl_certificate_authorities && @ssl_certificate_authorities.size > 1
51
+ fail LogStash::ConfigurationError, 'The `logstash` output supports at most one `ssl_certificate_authorities` path'
52
+ end
53
+
54
+ logger.debug("initializing inner HTTP output plugin")
55
+ @internal_http = plugin_factory.output('http').new(inner_http_output_options)
56
+ logger.debug("inner HTTP output plugin has been initialized")
57
+ end
58
+
59
+ def register
60
+ logger.debug("registering inner HTTP output plugin")
61
+ @internal_http.register
62
+ logger.debug("inner HTTP output plugin has been registered")
63
+ end
64
+
65
+ def multi_receive(events)
66
+ return if events.empty?
67
+ logger.trace("proxying #{events.size} events to inner HTTP plugin")
68
+ @internal_http.multi_receive(events)
69
+ rescue => e
70
+ logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
71
+ raise
72
+ end
73
+
74
+ def stop
75
+ logger.debug("stopping inner HTTP output plugin")
76
+ @internal_http.stop
77
+ logger.debug('inner HTTP output plugin has been stopped')
78
+ end
79
+
80
+ def close
81
+ logger.debug("closing inner HTTP output plugin")
82
+ @internal_http.close
83
+ logger.debug('inner HTTP output plugin has been closed')
84
+ end
85
+
86
+ def inner_http_output_options
87
+ @_inner_http_output_options ||= begin
88
+ http_options = {
89
+ 'url' => "#{@ssl_enabled ? 'https' : 'http'}://#{@host}:#{@port}",
90
+ 'http_method' => 'post',
91
+ 'retry_non_idempotent' => 'true',
92
+
93
+ # non-configurable codec
94
+ 'content_type' => 'application/x-ndjson',
95
+ 'format' => 'json_batch',
96
+ }
97
+
98
+ if @username
99
+ http_options['user'] = @username
100
+ http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
101
+ logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
102
+ elsif @password
103
+ fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
104
+ end
105
+
106
+ if @ssl_enabled == false
107
+ rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
108
+ fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
109
+ else
110
+ http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
111
+
112
+ http_options.merge!(ssl_identity_options)
113
+ http_options.merge!(ssl_trust_options)
114
+ end
115
+
116
+ http_options
117
+ end
118
+ end
119
+
120
+ def ssl_identity_options
121
+ if @ssl_certificate && @ssl_keystore_path
122
+ fail(LogStash::ConfigurationError, 'SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
123
+ elsif @ssl_certificate
124
+ return {
125
+ 'ssl_certificate' => @ssl_certificate,
126
+ 'ssl_key' => @ssl_key || fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided"),
127
+ }
128
+ elsif @ssl_key
129
+ fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
130
+ elsif @ssl_keystore_path
131
+ return {
132
+ 'ssl_keystore_path' => @ssl_keystore_path,
133
+ 'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
134
+ }
135
+ elsif @ssl_keystore_password
136
+ fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
137
+ else
138
+ return {}
139
+ end
140
+ end
141
+
142
+ def ssl_trust_options
143
+ {
144
+ 'ssl_verification_mode' => @ssl_verification_mode,
145
+ }.tap do |trust_options|
146
+ if @ssl_certificate_authorities&.any? && @ssl_truststore_path
147
+ fail(LogStash::ConfigurationError, 'SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both')
148
+ elsif @ssl_certificate_authorities&.any?
149
+ fail(LogStash::ConfigurationError, 'SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
150
+
151
+ trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities.first
152
+ elsif @ssl_truststore_path
153
+ fail(LogStash::ConfigurationError, 'SSL Truststore cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
154
+
155
+ trust_options['ssl_truststore_path'] = @ssl_truststore_path
156
+ trust_options['ssl_truststore_password'] = @ssl_truststore_password || fail(LogStash::ConfigurationError, '`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided')
157
+ elsif @ssl_truststore_password
158
+ fail(LogStash::ConfigurationError, '`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured')
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,36 @@
1
+ INTEGRATION_LOGSTASH_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(INTEGRATION_LOGSTASH_VERSION)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "logstash-integration-logstash"
5
+ s.version = INTEGRATION_LOGSTASH_VERSION
6
+ s.licenses = ["Apache-2.0"]
7
+ s.summary = "Collection of Logstash plugins that enable sending events from one Logstash pipeline to another"
8
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
9
+ s.authors = ["Elastic"]
10
+ s.email = "info@elastic.co"
11
+ s.homepage = "https://www.elastic.co/logstash"
12
+ s.platform = "java"
13
+ s.metadata = {
14
+ "logstash_plugin" => "true",
15
+ "logstash_group" => "integration",
16
+ "integration_plugins" => %w(
17
+ logstash-input-logstash
18
+ logstash-output-logstash
19
+ ).join(",")
20
+ }
21
+
22
+ s.require_paths = %w[lib vendor/jar-dependencies]
23
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "VERSION", "docs/**/*", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb"]
24
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
25
+
26
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
27
+ s.add_runtime_dependency "logstash-mixin-plugin_factory_support", "~> 1.0"
28
+ s.add_runtime_dependency "logstash-codec-json_lines", "~> 3.1"
29
+
30
+ s.add_runtime_dependency "logstash-input-http", ">= 3.7.0" # some params not available in older versions because they are renamed, such as `cacert` to `ssl_certificate_authorities`
31
+ s.add_runtime_dependency "logstash-output-http", ">= 5.6.0"
32
+
33
+ s.add_development_dependency "logstash-devutils"
34
+ s.add_development_dependency "rspec-collection_matchers"
35
+ s.add_development_dependency "random-port"
36
+ end