logstash-codec-avro 3.4.1-java → 3.5.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/docs/index.asciidoc +222 -0
- data/lib/logstash/codecs/avro.rb +248 -5
- data/logstash-codec-avro.gemspec +2 -1
- data/spec/integration/avro_integration_spec.rb +431 -0
- data/spec/integration/fixtures/jaas.config +5 -0
- data/spec/integration/fixtures/pwd +2 -0
- data/spec/integration/fixtures/trust-store_stub.jks +0 -0
- data/spec/integration/kafka_test_setup.sh +85 -0
- data/spec/integration/kafka_test_teardown.sh +16 -0
- data/spec/integration/setup_keystore_and_truststore.sh +17 -0
- data/spec/integration/start_auth_schema_registry.sh +8 -0
- data/spec/integration/start_schema_registry.sh +5 -0
- data/spec/integration/start_schema_registry_mutual.sh +5 -0
- data/spec/integration/stop_schema_registry.sh +6 -0
- data/spec/unit/avro_spec.rb +866 -0
- metadata +61 -22
- data/spec/codecs/avro_spec.rb +0 -203
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f88b7ec7b7a620275c87c502ae364c753450765dc059b0097c99ca1d21ef7d90
|
|
4
|
+
data.tar.gz: aa5749396621ee5061f10149e4f2d922bcf866421ff68065b2c84c28af60b65f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c09cd06cbd9a0b51d54786b05c584d08b6a2b43241554f41a7910378fe2cae9a14c74496b4d0232cb293a74bd8f9117c2b2de8c7afe34803f27504df56d5e256
|
|
7
|
+
data.tar.gz: d57d16ee2c57f4a98b64bf2a8556f48fc369d36db0ce93604727d24b5d69fc65d776c993ca6c2d222c4bba5607ff370b127c13232f7055829a9918c400beed9a
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
## 3.5.0
|
|
2
|
+
- Add SSL/TLS support for HTTPS schema registry connections
|
|
3
|
+
- Add `ssl_enabled` option to enable/disable SSL
|
|
4
|
+
- Add `ssl_certificate` and `ssl_key` options for PEM-based client authentication (unencrypted keys only)
|
|
5
|
+
- Add `ssl_certificate_authorities` option for PEM-based server certificate validation
|
|
6
|
+
- Add `ssl_verification_mode` option to control SSL verification (full, none)
|
|
7
|
+
- Add `ssl_cipher_suites` option to configure cipher suites
|
|
8
|
+
- Add `ssl_supported_protocols` option to configure TLS protocol versions (TLSv1.1, TLSv1.2, TLSv1.3)
|
|
9
|
+
- Add `ssl_truststore_path` and `ssl_truststore_password` options for server certificate validation (JKS/PKCS12)
|
|
10
|
+
- Add `ssl_keystore_path` and `ssl_keystore_password` options for mutual TLS authentication (JKS/PKCS12)
|
|
11
|
+
- Add `ssl_truststore_type` and `ssl_keystore_type` options (JKS or PKCS12)
|
|
12
|
+
- Add HTTP proxy support with `proxy` option
|
|
13
|
+
- Add HTTP basic authentication support with `username` and `password` options
|
|
14
|
+
|
|
1
15
|
## 3.4.1
|
|
2
16
|
- Fixes `(Errno::ENOENT) No such file or directory` error [#43](https://github.com/logstash-plugins/logstash-codec-avro/pull/43)
|
|
3
17
|
|
data/docs/index.asciidoc
CHANGED
|
@@ -83,9 +83,25 @@ output {
|
|
|
83
83
|
|Setting |Input type|Required
|
|
84
84
|
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
|
85
85
|
| <<plugins-{type}s-{plugin}-encoding>> | <<string,string>>, one of `["binary", "base64"]`|No
|
|
86
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
|
87
|
+
| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
|
|
86
88
|
| <<plugins-{type}s-{plugin}-schema_uri>> |<<string,string>>|Yes
|
|
89
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
|
|
90
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
|
|
91
|
+
| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |<<array,array>>|No
|
|
92
|
+
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
|
93
|
+
| <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
|
|
94
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
|
|
95
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
|
|
96
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
|
|
97
|
+
| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<array,array>>|No
|
|
98
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
|
|
99
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
|
|
100
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
|
|
101
|
+
| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
|
|
87
102
|
| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<boolean,boolean>>|No
|
|
88
103
|
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
|
104
|
+
| <<plugins-{type}s-{plugin}-username>> |<<string,string>>|No
|
|
89
105
|
|=======================================================================
|
|
90
106
|
|
|
91
107
|
|
|
@@ -112,6 +128,23 @@ Use `base64` (default) to indicate that this codec sends or expects to receive b
|
|
|
112
128
|
Set this option to `binary` to indicate that this codec sends or expects to receive binary Avro data.
|
|
113
129
|
|
|
114
130
|
|
|
131
|
+
[id="plugins-{type}s-{plugin}-password"]
|
|
132
|
+
===== `password`
|
|
133
|
+
|
|
134
|
+
* Value type is <<password,password>>
|
|
135
|
+
* There is no default value for this setting.
|
|
136
|
+
|
|
137
|
+
Password for HTTP basic authentication when fetching remote schemas.
|
|
138
|
+
Used together with `username`.
|
|
139
|
+
|
|
140
|
+
[id="plugins-{type}s-{plugin}-proxy"]
|
|
141
|
+
===== `proxy`
|
|
142
|
+
|
|
143
|
+
* Value type is <<uri,uri>>
|
|
144
|
+
* There is no default value for this setting.
|
|
145
|
+
|
|
146
|
+
The address of a forward HTTP proxy to use when contacting a remote schema registry.
|
|
147
|
+
|
|
115
148
|
[id="plugins-{type}s-{plugin}-schema_uri"]
|
|
116
149
|
===== `schema_uri`
|
|
117
150
|
|
|
@@ -134,6 +167,172 @@ example:
|
|
|
134
167
|
|
|
135
168
|
tag events with `_avroparsefailure` when decode fails
|
|
136
169
|
|
|
170
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
|
171
|
+
===== `ssl_certificate`
|
|
172
|
+
|
|
173
|
+
* Value type is <<path,path>>
|
|
174
|
+
* There is no default value for this setting.
|
|
175
|
+
|
|
176
|
+
Path to PEM encoded certificate file for client authentication (mutual TLS).
|
|
177
|
+
You may use this setting or <<plugins-{type}s-{plugin}-ssl_keystore_path>>, but not both simultaneously.
|
|
178
|
+
|
|
179
|
+
*Example*
|
|
180
|
+
[source,ruby]
|
|
181
|
+
----------------------------------
|
|
182
|
+
ssl_certificate => "/path/to/client.crt"
|
|
183
|
+
----------------------------------
|
|
184
|
+
|
|
185
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
|
186
|
+
===== `ssl_certificate_authorities`
|
|
187
|
+
|
|
188
|
+
* Value type is a list of <<path,path>>
|
|
189
|
+
* There is no default value for this setting.
|
|
190
|
+
|
|
191
|
+
Path to PEM encoded CA certificate file(s) for server verification.
|
|
192
|
+
This is an alternative to using <<plugins-{type}s-{plugin}-ssl_truststore_path>>.
|
|
193
|
+
You may use this setting or <<plugins-{type}s-{plugin}-ssl_truststore_path>>, but not both simultaneously.
|
|
194
|
+
|
|
195
|
+
*Example*
|
|
196
|
+
[source,ruby]
|
|
197
|
+
----------------------------------
|
|
198
|
+
ssl_certificate_authorities => ["/path/to/ca.crt"]
|
|
199
|
+
----------------------------------
|
|
200
|
+
|
|
201
|
+
[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
|
|
202
|
+
===== `ssl_cipher_suites`
|
|
203
|
+
|
|
204
|
+
* Value type is <<array,array>>
|
|
205
|
+
* There is no default value for this setting.
|
|
206
|
+
|
|
207
|
+
The list of cipher suites to use, listed by priorities.
|
|
208
|
+
Supported cipher suites vary depending on which version of Java is used.
|
|
209
|
+
|
|
210
|
+
[id="plugins-{type}s-{plugin}-ssl_key"]
|
|
211
|
+
===== `ssl_key`
|
|
212
|
+
|
|
213
|
+
* Value type is <<path,path>>
|
|
214
|
+
* There is no default value for this setting.
|
|
215
|
+
|
|
216
|
+
Path to PEM encoded private key file for client authentication.
|
|
217
|
+
Must be used together with <<plugins-{type}s-{plugin}-ssl_certificate>>.
|
|
218
|
+
The private key must be unencrypted (passphrase-protected keys are not supported).
|
|
219
|
+
|
|
220
|
+
*Example*
|
|
221
|
+
[source,ruby]
|
|
222
|
+
----------------------------------
|
|
223
|
+
ssl_key => "/path/to/client.key"
|
|
224
|
+
----------------------------------
|
|
225
|
+
|
|
226
|
+
[id="plugins-{type}s-{plugin}-ssl_enabled"]
|
|
227
|
+
===== `ssl_enabled`
|
|
228
|
+
|
|
229
|
+
* Value type is <<boolean,boolean>>
|
|
230
|
+
* There is no default value for this setting.
|
|
231
|
+
|
|
232
|
+
Enable SSL/TLS secured communication to remote schema registry.
|
|
233
|
+
When using HTTPS schema URIs, SSL is automatically enabled.
|
|
234
|
+
|
|
235
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
|
|
236
|
+
===== `ssl_keystore_path`
|
|
237
|
+
|
|
238
|
+
* Value type is <<path,path>>
|
|
239
|
+
* There is no default value for this setting.
|
|
240
|
+
|
|
241
|
+
The path to the JKS or PKCS12 keystore file for client certificate authentication.
|
|
242
|
+
Use this when the schema registry requires mutual TLS (mTLS) authentication.
|
|
243
|
+
|
|
244
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
|
|
245
|
+
===== `ssl_keystore_password`
|
|
246
|
+
|
|
247
|
+
* Value type is <<password,password>>
|
|
248
|
+
* There is no default value for this setting.
|
|
249
|
+
|
|
250
|
+
The password for the keystore file specified in <<plugins-{type}s-{plugin}-ssl_keystore_path>>.
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
|
|
254
|
+
===== `ssl_keystore_type`
|
|
255
|
+
|
|
256
|
+
* Value type is <<string,string>>
|
|
257
|
+
* There is no default value for this setting.
|
|
258
|
+
|
|
259
|
+
The format of the keystore file. It must be either `jks` or `pkcs12`.
|
|
260
|
+
|
|
261
|
+
[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
|
|
262
|
+
===== `ssl_supported_protocols`
|
|
263
|
+
|
|
264
|
+
* Value type is <<array,array>>
|
|
265
|
+
* Default value is `[]` (uses Java defaults)
|
|
266
|
+
* Valid values are: `TLSv1.1`, `TLSv1.2`, `TLSv1.3`
|
|
267
|
+
|
|
268
|
+
List of allowed SSL/TLS protocol versions.
|
|
269
|
+
When not specified, the JVM defaults are used.
|
|
270
|
+
|
|
271
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
|
|
272
|
+
===== `ssl_truststore_path`
|
|
273
|
+
|
|
274
|
+
* Value type is <<path,path>>
|
|
275
|
+
* There is no default value for this setting.
|
|
276
|
+
|
|
277
|
+
The path to the JKS or PKCS12 truststore file containing certificates to verify
|
|
278
|
+
the schema registry server's certificate.
|
|
279
|
+
|
|
280
|
+
*Example*
|
|
281
|
+
[source,ruby]
|
|
282
|
+
----------------------------------
|
|
283
|
+
input {
|
|
284
|
+
kafka {
|
|
285
|
+
codec => avro {
|
|
286
|
+
schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1"
|
|
287
|
+
ssl_truststore_path => "/path/to/truststore.jks"
|
|
288
|
+
ssl_truststore_password => "${TRUSTSTORE_PASSWORD}"
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
----------------------------------
|
|
293
|
+
|
|
294
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
|
|
295
|
+
===== `ssl_truststore_password`
|
|
296
|
+
|
|
297
|
+
* Value type is <<password,password>>
|
|
298
|
+
* There is no default value for this setting.
|
|
299
|
+
|
|
300
|
+
The password for the truststore file specified in <<plugins-{type}s-{plugin}-ssl_truststore_path>>.
|
|
301
|
+
|
|
302
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
|
|
303
|
+
===== `ssl_truststore_type`
|
|
304
|
+
|
|
305
|
+
* Value type is <<string,string>>
|
|
306
|
+
* There is no default value for this setting.
|
|
307
|
+
|
|
308
|
+
The format of the truststore file. It must be either `jks` or `pkcs12`.
|
|
309
|
+
|
|
310
|
+
[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
|
|
311
|
+
===== `ssl_verification_mode`
|
|
312
|
+
|
|
313
|
+
* Value type is <<string,string>>
|
|
314
|
+
* Default value is `"full"`
|
|
315
|
+
* Valid options are: `full`, `none`
|
|
316
|
+
|
|
317
|
+
Options to verify the server's certificate:
|
|
318
|
+
|
|
319
|
+
* `full`: Validates that the provided certificate has an issue date that's within the not_before and not_after dates; chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. (recommended)
|
|
320
|
+
* `none`: Performs no certificate validation. **Warning:** Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
|
|
321
|
+
|
|
322
|
+
*Example*
|
|
323
|
+
[source,ruby]
|
|
324
|
+
----------------------------------
|
|
325
|
+
input {
|
|
326
|
+
kafka {
|
|
327
|
+
codec => avro {
|
|
328
|
+
schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1"
|
|
329
|
+
ssl_certificate_authorities => ["/path/to/ca.crt"]
|
|
330
|
+
ssl_verification_mode => "full"
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
----------------------------------
|
|
335
|
+
|
|
137
336
|
[id="plugins-{type}s-{plugin}-target"]
|
|
138
337
|
===== `target`
|
|
139
338
|
|
|
@@ -156,3 +355,26 @@ input {
|
|
|
156
355
|
}
|
|
157
356
|
}
|
|
158
357
|
----------------------------------
|
|
358
|
+
|
|
359
|
+
[id="plugins-{type}s-{plugin}-username"]
|
|
360
|
+
===== `username`
|
|
361
|
+
|
|
362
|
+
* Value type is <<string,string>>
|
|
363
|
+
* There is no default value for this setting.
|
|
364
|
+
|
|
365
|
+
Username for HTTP basic authentication when fetching remote schemas.
|
|
366
|
+
Used together with `password`.
|
|
367
|
+
|
|
368
|
+
*Example*
|
|
369
|
+
[source,ruby]
|
|
370
|
+
----------------------------------
|
|
371
|
+
input {
|
|
372
|
+
kafka {
|
|
373
|
+
codec => avro {
|
|
374
|
+
schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1"
|
|
375
|
+
username => "registry_user"
|
|
376
|
+
password => "${REGISTRY_PASSWORD}"
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
----------------------------------
|
data/lib/logstash/codecs/avro.rb
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
2
|
require "open-uri"
|
|
3
|
+
require "manticore"
|
|
3
4
|
require "avro"
|
|
4
5
|
require "base64"
|
|
6
|
+
require "json"
|
|
5
7
|
require "logstash/codecs/base"
|
|
6
8
|
require "logstash/event"
|
|
7
9
|
require "logstash/timestamp"
|
|
@@ -50,6 +52,17 @@ require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
|
|
50
52
|
# }
|
|
51
53
|
# ----------------------------------
|
|
52
54
|
class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
55
|
+
class BadResponseCodeError < LogStash::Error
|
|
56
|
+
attr_reader :code, :message, :uri
|
|
57
|
+
|
|
58
|
+
def initialize(code, message, uri)
|
|
59
|
+
@code = code
|
|
60
|
+
@message = message
|
|
61
|
+
@uri = uri
|
|
62
|
+
super("HTTP #{code}: #{message} (#{uri})")
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
53
66
|
config_name "avro"
|
|
54
67
|
|
|
55
68
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
|
@@ -84,9 +97,59 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
|
84
97
|
# NOTE: the target is only relevant while decoding data into a new event.
|
|
85
98
|
config :target, :validate => :field_reference
|
|
86
99
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
100
|
+
# Proxy server URL for schema registry connections
|
|
101
|
+
config :proxy, :validate => :uri
|
|
102
|
+
|
|
103
|
+
# Username for HTTP basic authentication
|
|
104
|
+
config :username, :validate => :string
|
|
105
|
+
|
|
106
|
+
# Password for HTTP basic authentication
|
|
107
|
+
config :password, :validate => :password
|
|
108
|
+
|
|
109
|
+
# Enable SSL/TLS secured communication to remote schema registry
|
|
110
|
+
config :ssl_enabled, :validate => :boolean
|
|
111
|
+
|
|
112
|
+
# PEM-based SSL configuration (alternative to keystore/truststore)
|
|
113
|
+
# Path to PEM encoded certificate file for client authentication
|
|
114
|
+
config :ssl_certificate, :validate => :path
|
|
115
|
+
|
|
116
|
+
# Path to PEM encoded private key file for client authentication
|
|
117
|
+
config :ssl_key, :validate => :path
|
|
118
|
+
|
|
119
|
+
# Path to PEM encoded CA certificate file(s) for server verification
|
|
120
|
+
# Can be a single file or directory containing multiple CA certificates
|
|
121
|
+
config :ssl_certificate_authorities, :validate => :path, :list => true
|
|
122
|
+
|
|
123
|
+
# Options to verify the server's certificate.
|
|
124
|
+
# "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
|
|
125
|
+
# chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
|
|
126
|
+
# "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
|
|
127
|
+
config :ssl_verification_mode, :validate => %w[full none]
|
|
128
|
+
|
|
129
|
+
# The keystore path
|
|
130
|
+
config :ssl_keystore_path, :validate => :path
|
|
131
|
+
|
|
132
|
+
# The keystore password
|
|
133
|
+
config :ssl_keystore_password, :validate => :password
|
|
134
|
+
|
|
135
|
+
# Keystore type (jks or pkcs12)
|
|
136
|
+
config :ssl_keystore_type, :validate => %w[jks pkcs12]
|
|
137
|
+
|
|
138
|
+
# The truststore path
|
|
139
|
+
config :ssl_truststore_path, :validate => :path
|
|
140
|
+
|
|
141
|
+
# The truststore password
|
|
142
|
+
config :ssl_truststore_password, :validate => :password
|
|
143
|
+
|
|
144
|
+
# Truststore type (jks or pkcs12)
|
|
145
|
+
config :ssl_truststore_type, :validate => %w[jks pkcs12]
|
|
146
|
+
|
|
147
|
+
# The list of cipher suites to use, listed by priorities.
|
|
148
|
+
# Supported cipher suites vary depending on which version of Java is used.
|
|
149
|
+
config :ssl_cipher_suites, :validate => :string, :list => true
|
|
150
|
+
|
|
151
|
+
# SSL supported protocols
|
|
152
|
+
config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :list => true
|
|
90
153
|
|
|
91
154
|
public
|
|
92
155
|
def initialize(*params)
|
|
@@ -95,7 +158,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
|
95
158
|
end
|
|
96
159
|
|
|
97
160
|
def register
|
|
98
|
-
@schema = Avro::Schema.parse(
|
|
161
|
+
@schema = Avro::Schema.parse(fetch_schema(schema_uri))
|
|
99
162
|
end
|
|
100
163
|
|
|
101
164
|
public
|
|
@@ -129,6 +192,186 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
|
129
192
|
@on_event.call(event, Base64.strict_encode64(buffer.string))
|
|
130
193
|
else
|
|
131
194
|
@on_event.call(event, buffer.string)
|
|
132
|
-
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
private
|
|
199
|
+
def fetch_schema(uri_string)
|
|
200
|
+
http_connection = uri_string.start_with?('http://')
|
|
201
|
+
https_connection = uri_string.start_with?('https://')
|
|
202
|
+
|
|
203
|
+
if http_connection
|
|
204
|
+
ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") && k != "ssl_enabled" }
|
|
205
|
+
if ssl_config_provided.any?
|
|
206
|
+
raise_config_error! "When SSL is disabled, the following provided parameters are not allowed: #{ssl_config_provided}"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
credentials_configured = @username && @password
|
|
210
|
+
@logger.warn("Credentials are being sent over unencrypted HTTP. This may bring security risk.") if credentials_configured
|
|
211
|
+
fetch_remote_schema(uri_string)
|
|
212
|
+
elsif https_connection
|
|
213
|
+
validate_ssl_settings!
|
|
214
|
+
fetch_remote_schema(uri_string)
|
|
215
|
+
else
|
|
216
|
+
# local schema
|
|
217
|
+
URI.open(uri_string, &:read)
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def fetch_remote_schema(uri_string)
|
|
222
|
+
client = nil
|
|
223
|
+
client_options = {}
|
|
224
|
+
|
|
225
|
+
if @proxy && !@proxy.empty?
|
|
226
|
+
client_options[:proxy] = @proxy.to_s
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
basic_auth_options = build_basic_auth
|
|
230
|
+
client_options[:auth] = basic_auth_options unless basic_auth_options.empty?
|
|
231
|
+
|
|
232
|
+
if @ssl_enabled
|
|
233
|
+
ssl_options = build_ssl_options
|
|
234
|
+
client_options[:ssl] = ssl_options unless ssl_options.empty?
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
client = Manticore::Client.new(client_options)
|
|
238
|
+
|
|
239
|
+
@logger.debug("Fetching schema from #{uri_string}")
|
|
240
|
+
|
|
241
|
+
max_retries = 3
|
|
242
|
+
retry_count = 0
|
|
243
|
+
body = nil
|
|
244
|
+
|
|
245
|
+
begin
|
|
246
|
+
response = client.get(uri_string).call
|
|
247
|
+
|
|
248
|
+
unless response.code == 200
|
|
249
|
+
@logger.error("Failed to fetch schema from #{uri_string}: #{response.code} - #{response.message}")
|
|
250
|
+
raise BadResponseCodeError.new(response.code, response.message, uri_string)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
body = response.body
|
|
254
|
+
|
|
255
|
+
# Parse and extract schema
|
|
256
|
+
parsed = JSON.parse(body)
|
|
257
|
+
return parsed.is_a?(Hash) && parsed.has_key?('schema') ? parsed['schema'] : body
|
|
258
|
+
|
|
259
|
+
rescue JSON::ParserError
|
|
260
|
+
return body
|
|
261
|
+
rescue Manticore::ManticoreException, BadResponseCodeError => e
|
|
262
|
+
# 4xx don't retry
|
|
263
|
+
if e.is_a?(BadResponseCodeError) && e.code >= 400 && e.code < 500
|
|
264
|
+
@logger.error("Failed to fetch schema from #{uri_string}: #{e.code} - #{e.message}")
|
|
265
|
+
raise
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# retry block
|
|
269
|
+
retry_count += 1
|
|
270
|
+
if retry_count < max_retries
|
|
271
|
+
backoff_time = 2 ** (retry_count) # Exponential backoff: 1s, 2s, 4s
|
|
272
|
+
@logger.warn("Attempt #{retry_count}/#{max_retries} failed for #{uri_string}: #{e.class} - #{e.message}. Retrying in #{backoff_time}s...")
|
|
273
|
+
sleep(backoff_time)
|
|
274
|
+
retry
|
|
275
|
+
else
|
|
276
|
+
@logger.error("Failed to fetch schema from #{uri_string} after #{max_retries} attempts: #{e.class} - #{e.message}")
|
|
277
|
+
raise
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
ensure
|
|
281
|
+
client.close if client
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def build_basic_auth
|
|
285
|
+
if !@username && !@password
|
|
286
|
+
return {}
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
raise LogStash::ConfigurationError, "`username` requires `password`" if @username && !@password
|
|
290
|
+
raise LogStash::ConfigurationError, "`password` is not allowed unless `username` is specified" if !@username && @password
|
|
291
|
+
|
|
292
|
+
raise LogStash::ConfigurationError, "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty?
|
|
293
|
+
|
|
294
|
+
{:user => @username, :password => @password.value}
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def validate_ssl_settings!
|
|
298
|
+
@ssl_enabled = true if @ssl_enabled.nil?
|
|
299
|
+
raise_config_error! "Secured #{@schema_uri} connection requires `ssl_enabled => true`. " unless @ssl_enabled
|
|
300
|
+
@ssl_verification_mode = "full".freeze if @ssl_verification_mode.nil?
|
|
301
|
+
|
|
302
|
+
# optional: presenting our identity
|
|
303
|
+
raise_config_error! "`ssl_certificate` and `ssl_keystore_path` cannot be used together." if @ssl_certificate && @ssl_keystore_path
|
|
304
|
+
raise_config_error! "`ssl_certificate` requires `ssl_key`" if @ssl_certificate && !@ssl_key
|
|
305
|
+
ensure_readable_and_non_writable! "ssl_certificate", @ssl_certificate if @ssl_certificate
|
|
306
|
+
|
|
307
|
+
raise_config_error! "`ssl_key` is not allowed unless `ssl_certificate` is specified" if @ssl_key && !@ssl_certificate
|
|
308
|
+
ensure_readable_and_non_writable! "ssl_key", @ssl_key if @ssl_key
|
|
309
|
+
|
|
310
|
+
raise_config_error! "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_password && !@ssl_keystore_path
|
|
311
|
+
raise_config_error! "`ssl_keystore_password` cannot be empty" if @ssl_keystore_password && @ssl_keystore_password.value.empty?
|
|
312
|
+
raise_config_error! "`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_type && !@ssl_keystore_path
|
|
313
|
+
|
|
314
|
+
ensure_readable_and_non_writable! "ssl_keystore_path", @ssl_keystore_path if @ssl_keystore_path
|
|
315
|
+
|
|
316
|
+
# establishing trust of the server we connect to
|
|
317
|
+
# system-provided trust requires verification mode enabled
|
|
318
|
+
if @ssl_verification_mode == "none"
|
|
319
|
+
raise_config_error! "`ssl_truststore_path` requires `ssl_verification_mode` to be `full`" if @ssl_truststore_path
|
|
320
|
+
raise_config_error! "`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`" if @ssl_truststore_password
|
|
321
|
+
raise_config_error! "`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`" if @ssl_certificate_authorities
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
raise_config_error! "`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together." if @ssl_truststore_path && @ssl_certificate_authorities
|
|
325
|
+
ensure_readable_and_non_writable! "ssl_truststore_path", @ssl_truststore_path if @ssl_truststore_path
|
|
326
|
+
|
|
327
|
+
raise_config_error! "`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified" if !@ssl_truststore_path && @ssl_truststore_password
|
|
328
|
+
raise_config_error! "`ssl_truststore_password` cannot be empty" if @ssl_truststore_password && @ssl_truststore_password.value.empty?
|
|
329
|
+
|
|
330
|
+
if !@ssl_truststore_path && @ssl_certificate_authorities&.empty?
|
|
331
|
+
raise_config_error! "`ssl_certificate_authorities` cannot be empty"
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
if @ssl_certificate_authorities && !@ssl_certificate_authorities.empty?
|
|
335
|
+
raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities.size > 1
|
|
336
|
+
ensure_readable_and_non_writable! "ssl_certificate_authorities", @ssl_certificate_authorities.first
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def build_ssl_options
|
|
341
|
+
ssl_options = {}
|
|
342
|
+
|
|
343
|
+
ssl_options[:client_cert] = @ssl_certificate if @ssl_certificate
|
|
344
|
+
ssl_options[:client_key] = @ssl_key if @ssl_key
|
|
345
|
+
|
|
346
|
+
ssl_options[:ca_file] = @ssl_certificate_authorities&.first if @ssl_certificate_authorities
|
|
347
|
+
|
|
348
|
+
ssl_options[:cipher_suites] = @ssl_cipher_suites if @ssl_cipher_suites
|
|
349
|
+
|
|
350
|
+
ssl_options[:verify] = :default if @ssl_verification_mode == 'full'
|
|
351
|
+
ssl_options[:verify] = :disable if @ssl_verification_mode == 'none'
|
|
352
|
+
|
|
353
|
+
ssl_options[:keystore] = @ssl_keystore_path if @ssl_keystore_path
|
|
354
|
+
ssl_options[:keystore_password] = @ssl_keystore_password&.value if @ssl_keystore_path && @ssl_keystore_password
|
|
355
|
+
ssl_options[:keystore_type] = @ssl_keystore_type.downcase if @ssl_keystore_path && @ssl_keystore_type
|
|
356
|
+
|
|
357
|
+
ssl_options[:truststore] = @ssl_truststore_path if @ssl_truststore_path
|
|
358
|
+
ssl_options[:truststore_password] = @ssl_truststore_password&.value if @ssl_truststore_path && @ssl_truststore_password
|
|
359
|
+
ssl_options[:truststore_type] = @ssl_truststore_type.downcase if @ssl_truststore_path && @ssl_truststore_type
|
|
360
|
+
|
|
361
|
+
ssl_options[:protocols] = @ssl_supported_protocols if @ssl_supported_protocols && @ssl_supported_protocols&.any?
|
|
362
|
+
|
|
363
|
+
ssl_options
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
##
|
|
367
|
+
# @param message [String]
|
|
368
|
+
# @raise [LogStash::ConfigurationError]
|
|
369
|
+
def raise_config_error!(message)
|
|
370
|
+
raise LogStash::ConfigurationError, message
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def ensure_readable_and_non_writable!(name, path)
|
|
374
|
+
raise_config_error! "Specified #{name} #{path} path must be readable." unless File.readable?(path)
|
|
375
|
+
raise_config_error! "Specified #{name} #{path} path must not be writable." if File.writable?(path)
|
|
133
376
|
end
|
|
134
377
|
end
|
data/logstash-codec-avro.gemspec
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
|
|
3
3
|
s.name = 'logstash-codec-avro'
|
|
4
|
-
s.version = '3.
|
|
4
|
+
s.version = '3.5.0'
|
|
5
5
|
s.platform = 'java'
|
|
6
6
|
s.licenses = ['Apache-2.0']
|
|
7
7
|
s.summary = "Reads serialized Avro records as Logstash events"
|
|
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
|
|
|
23
23
|
# Gem dependencies
|
|
24
24
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
|
25
25
|
s.add_runtime_dependency "avro", "~> 1.10.2" #(Apache 2.0 license)
|
|
26
|
+
s.add_runtime_dependency "manticore", '>= 0.8.0', '< 1.0.0'
|
|
26
27
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
|
|
27
28
|
s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
|
|
28
29
|
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|