logstash-codec-avro 3.4.1-java → 3.5.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/docs/index.asciidoc +222 -0
- data/lib/logstash/codecs/avro.rb +248 -5
- data/logstash-codec-avro.gemspec +2 -1
- data/spec/integration/avro_integration_spec.rb +431 -0
- data/spec/integration/fixtures/jaas.config +5 -0
- data/spec/integration/fixtures/pwd +2 -0
- data/spec/integration/fixtures/trust-store_stub.jks +0 -0
- data/spec/integration/kafka_test_setup.sh +85 -0
- data/spec/integration/kafka_test_teardown.sh +16 -0
- data/spec/integration/setup_keystore_and_truststore.sh +17 -0
- data/spec/integration/start_auth_schema_registry.sh +8 -0
- data/spec/integration/start_schema_registry.sh +5 -0
- data/spec/integration/start_schema_registry_mutual.sh +5 -0
- data/spec/integration/stop_schema_registry.sh +6 -0
- data/spec/unit/avro_spec.rb +866 -0
- metadata +61 -22
- data/spec/codecs/avro_spec.rb +0 -203
|
@@ -0,0 +1,866 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require 'logstash/devutils/rspec/spec_helper'
|
|
3
|
+
require 'insist'
|
|
4
|
+
require 'avro'
|
|
5
|
+
require 'base64'
|
|
6
|
+
require 'logstash/codecs/avro'
|
|
7
|
+
require 'logstash/event'
|
|
8
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
|
9
|
+
|
|
10
|
+
describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures do
|
|
11
|
+
let(:paths) do
|
|
12
|
+
{
|
|
13
|
+
# path has to be created, otherwise config :path validation fails
|
|
14
|
+
# and since we cannot control the chmod operations on paths, we should stub file readable? and writable? operations
|
|
15
|
+
:test_path => "spec/unit/resources/do_not_remove_path"
|
|
16
|
+
}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
|
20
|
+
before(:each) do
|
|
21
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
context "non binary data" do
|
|
25
|
+
let (:avro_config) {{ 'schema_uri' => '
|
|
26
|
+
{"type": "record", "name": "Test",
|
|
27
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
|
28
|
+
{"name": "bar", "type": "int"}]}' }}
|
|
29
|
+
let (:test_event_hash) { { "foo" => "hello", "bar" => 10 } }
|
|
30
|
+
let (:test_event) {LogStash::Event.new(test_event_hash)}
|
|
31
|
+
|
|
32
|
+
subject do
|
|
33
|
+
allow_any_instance_of(LogStash::Codecs::Avro).to \
|
|
34
|
+
receive(:fetch_schema).and_return(avro_config['schema_uri'])
|
|
35
|
+
next LogStash::Codecs::Avro.new(avro_config)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
context "#decode" do
|
|
39
|
+
it "should return an LogStash::Event from raw and base64 encoded avro data" do
|
|
40
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
41
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
42
|
+
buffer = StringIO.new
|
|
43
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
44
|
+
dw.write(test_event.to_hash, encoder)
|
|
45
|
+
|
|
46
|
+
subject.decode(Base64.strict_encode64(buffer.string)) do |event|
|
|
47
|
+
insist {event.is_a? LogStash::Event}
|
|
48
|
+
insist {event.get("foo")} == test_event.get("foo")
|
|
49
|
+
insist {event.get("bar")} == test_event.get("bar")
|
|
50
|
+
expect(event.get('[event][original]')).to eq(Base64.strict_encode64(buffer.string)) if ecs_compatibility != :disabled
|
|
51
|
+
end
|
|
52
|
+
subject.decode(buffer.string) do |event|
|
|
53
|
+
insist {event.is_a? LogStash::Event}
|
|
54
|
+
insist {event.get("foo")} == test_event.get("foo")
|
|
55
|
+
insist {event.get("bar")} == test_event.get("bar")
|
|
56
|
+
expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
it "should throw exception if decoding fails" do
|
|
61
|
+
expect {subject.decode("not avro") {|_| }}.to raise_error NoMethodError
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
context "with binary encoding" do
|
|
66
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
|
67
|
+
|
|
68
|
+
it "should return an LogStash::Event from raw and base64 encoded avro data" do
|
|
69
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
70
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
71
|
+
buffer = StringIO.new
|
|
72
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
73
|
+
dw.write(test_event.to_hash, encoder)
|
|
74
|
+
|
|
75
|
+
subject.decode(buffer.string) do |event|
|
|
76
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
|
77
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
|
78
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
|
79
|
+
expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
it "should raise an error if base64 encoded data is provided" do
|
|
84
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
85
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
86
|
+
buffer = StringIO.new
|
|
87
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
88
|
+
dw.write(test_event.to_hash, encoder)
|
|
89
|
+
|
|
90
|
+
expect {subject.decode(Base64.strict_encode64(buffer.string))}.to raise_error
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
context "#decode with tag_on_failure" do
|
|
95
|
+
let (:avro_config) {{ 'schema_uri' => '
|
|
96
|
+
{"type": "record", "name": "Test",
|
|
97
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
|
98
|
+
{"name": "bar", "type": "int"}]}',
|
|
99
|
+
'tag_on_failure' => true}}
|
|
100
|
+
|
|
101
|
+
it "should tag event on failure" do
|
|
102
|
+
subject.decode("not avro") do |event|
|
|
103
|
+
insist {event.is_a? LogStash::Event}
|
|
104
|
+
insist {event.get("tags")} == ["_avroparsefailure"]
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
context "#decode with target" do
|
|
110
|
+
let(:avro_target) { "avro_target" }
|
|
111
|
+
let (:avro_config) {{ 'schema_uri' => '
|
|
112
|
+
{"type": "record", "name": "Test",
|
|
113
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
|
114
|
+
{"name": "bar", "type": "int"}]}',
|
|
115
|
+
'target' => avro_target}}
|
|
116
|
+
|
|
117
|
+
it "should return an LogStash::Event with content in target" do
|
|
118
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
119
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
120
|
+
buffer = StringIO.new
|
|
121
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
122
|
+
dw.write(test_event.to_hash, encoder)
|
|
123
|
+
|
|
124
|
+
subject.decode(buffer.string) do |event|
|
|
125
|
+
insist {event.get("[#{avro_target}][foo]")} == test_event.get("foo")
|
|
126
|
+
insist {event.get("[#{avro_target}][bar]")} == test_event.get("bar")
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
context "#encode" do
|
|
132
|
+
it "should return avro data from a LogStash::Event" do
|
|
133
|
+
got_event = false
|
|
134
|
+
subject.on_event do |event, data|
|
|
135
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
136
|
+
datum = StringIO.new(Base64.strict_decode64(data))
|
|
137
|
+
decoder = Avro::IO::BinaryDecoder.new(datum)
|
|
138
|
+
datum_reader = Avro::IO::DatumReader.new(schema)
|
|
139
|
+
record = datum_reader.read(decoder)
|
|
140
|
+
|
|
141
|
+
insist {record["foo"]} == test_event.get("foo")
|
|
142
|
+
insist {record["bar"]} == test_event.get("bar")
|
|
143
|
+
insist {event.is_a? LogStash::Event}
|
|
144
|
+
got_event = true
|
|
145
|
+
end
|
|
146
|
+
subject.encode(test_event)
|
|
147
|
+
insist {got_event}
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
context "with binary encoding" do
|
|
151
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
|
152
|
+
|
|
153
|
+
it "should return avro data from a LogStash::Event not base64 encoded" do
|
|
154
|
+
got_event = false
|
|
155
|
+
subject.on_event do |event, data|
|
|
156
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
157
|
+
datum = StringIO.new(data)
|
|
158
|
+
decoder = Avro::IO::BinaryDecoder.new(datum)
|
|
159
|
+
datum_reader = Avro::IO::DatumReader.new(schema)
|
|
160
|
+
record = datum_reader.read(decoder)
|
|
161
|
+
|
|
162
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
|
163
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
|
164
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
|
165
|
+
got_event = true
|
|
166
|
+
end
|
|
167
|
+
subject.encode(test_event)
|
|
168
|
+
expect(got_event).to be true
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
context "binary data" do
|
|
173
|
+
|
|
174
|
+
let (:avro_config) {{ 'schema_uri' => '{"namespace": "com.systems.test.data",
|
|
175
|
+
"type": "record",
|
|
176
|
+
"name": "TestRecord",
|
|
177
|
+
"fields": [
|
|
178
|
+
{"name": "name", "type": ["string", "null"]},
|
|
179
|
+
{"name": "longitude", "type": ["double", "null"]},
|
|
180
|
+
{"name": "latitude", "type": ["double", "null"]}
|
|
181
|
+
]
|
|
182
|
+
}' }}
|
|
183
|
+
let (:test_event) {LogStash::Event.new({ "name" => "foo", "longitude" => 21.01234.to_f, "latitude" => 111.0123.to_f })}
|
|
184
|
+
|
|
185
|
+
subject do
|
|
186
|
+
allow_any_instance_of(LogStash::Codecs::Avro).to \
|
|
187
|
+
receive(:fetch_schema).and_return(avro_config['schema_uri'])
|
|
188
|
+
next LogStash::Codecs::Avro.new(avro_config)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
it "should correctly encode binary data" do
|
|
192
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
|
193
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
194
|
+
buffer = StringIO.new
|
|
195
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
196
|
+
dw.write(test_event.to_hash, encoder)
|
|
197
|
+
|
|
198
|
+
subject.decode(Base64.strict_encode64(buffer.string)) do |event|
|
|
199
|
+
insist {event.is_a? LogStash::Event}
|
|
200
|
+
insist {event.get("name")} == test_event.get("name")
|
|
201
|
+
insist {event.get("longitude")} == test_event.get("longitude")
|
|
202
|
+
insist {event.get("latitude")} == test_event.get("latitude")
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
context "remote schema registry" do
|
|
212
|
+
let(:test_schema) do
|
|
213
|
+
'{"type": "record", "name": "Test",
|
|
214
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
|
215
|
+
{"name": "bar", "type": "int"}]}'
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
subject do
|
|
219
|
+
allow_any_instance_of(LogStash::Codecs::Avro).to receive(:fetch_schema).and_return(test_schema)
|
|
220
|
+
next LogStash::Codecs::Avro.new(avro_config)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
context "basic authentication" do
|
|
224
|
+
|
|
225
|
+
context "with both username and password" do
|
|
226
|
+
let(:avro_config) do
|
|
227
|
+
{
|
|
228
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
229
|
+
'username' => 'test_user',
|
|
230
|
+
'password' => 'test_&^%$!password'
|
|
231
|
+
}
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
it "uses user and password" do
|
|
235
|
+
auth = subject.send(:build_basic_auth)
|
|
236
|
+
expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'})
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
it "includes valid credentials in auth hash" do
|
|
240
|
+
auth = subject.send(:build_basic_auth)
|
|
241
|
+
expect(auth[:user]).not_to be_empty
|
|
242
|
+
expect(auth[:password]).not_to be_empty
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
context "with only username" do
|
|
247
|
+
let(:avro_config) do
|
|
248
|
+
{
|
|
249
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
250
|
+
'username' => 'test_user'
|
|
251
|
+
}
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
it "raises ConfigurationError" do
|
|
255
|
+
expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`username` requires `password`/)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
context "with only password" do
|
|
260
|
+
let(:avro_config) do
|
|
261
|
+
{
|
|
262
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
263
|
+
'password' => 'test_&^%$!password'
|
|
264
|
+
}
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
it "raises ConfigurationError" do
|
|
268
|
+
expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`password` is not allowed unless `username` is specified/)
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
context "with empty username" do
|
|
273
|
+
let(:avro_config) do
|
|
274
|
+
{
|
|
275
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
276
|
+
'username' => '',
|
|
277
|
+
'password' => 'test_&^%$!password'
|
|
278
|
+
}
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
it "raises ConfigurationError" do
|
|
282
|
+
expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/)
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
context "with empty password" do
|
|
287
|
+
let(:avro_config) do
|
|
288
|
+
{
|
|
289
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
290
|
+
'username' => 'test_user',
|
|
291
|
+
'password' => ''
|
|
292
|
+
}
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
it "raises ConfigurationError" do
|
|
296
|
+
expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/)
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
context "with neither username nor password" do
|
|
301
|
+
let(:avro_config) do
|
|
302
|
+
{
|
|
303
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc'
|
|
304
|
+
}
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
it "returns empty hash" do
|
|
308
|
+
auth = subject.send(:build_basic_auth)
|
|
309
|
+
expect(auth).to be_empty
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
context "with unsecure connection and credentials" do
|
|
314
|
+
let(:avro_config) do
|
|
315
|
+
{
|
|
316
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
317
|
+
'username' => 'test_user',
|
|
318
|
+
'password' => 'test_&^%$!password'
|
|
319
|
+
}
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
it "still returns valid auth hash" do
|
|
323
|
+
allow(subject.logger).to receive(:warn)
|
|
324
|
+
auth = subject.send(:build_basic_auth)
|
|
325
|
+
expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'})
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
context "secured connection against schema registry" do
|
|
331
|
+
|
|
332
|
+
before do
|
|
333
|
+
allow(File).to receive(:readable?).and_return(true)
|
|
334
|
+
allow(File).to receive(:writable?).and_return(false)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
context "explicit and inferred SSL" do
|
|
338
|
+
context "with explicit ssl_enabled => true" do
|
|
339
|
+
let(:avro_config) do
|
|
340
|
+
{
|
|
341
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
342
|
+
'ssl_enabled' => true
|
|
343
|
+
}
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
it "enables SSL" do
|
|
347
|
+
expect(subject.instance_variable_get(:@ssl_enabled)).to be true
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
context "with HTTPS URI (inferred ssl_enabled)" do
|
|
352
|
+
let(:avro_config) do
|
|
353
|
+
{
|
|
354
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc'
|
|
355
|
+
}
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
it "automatically enables SSL for HTTPS URIs" do
|
|
359
|
+
subject.send(:validate_ssl_settings!)
|
|
360
|
+
expect(subject.instance_variable_get(:@ssl_enabled)).to be true
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
context "with explicit ssl_enabled => false" do
|
|
365
|
+
let(:avro_config) do
|
|
366
|
+
{
|
|
367
|
+
'schema_uri' => 'http://schema-registry.example.com/schema.avsc',
|
|
368
|
+
'ssl_enabled' => false
|
|
369
|
+
}
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
it "disables SSL" do
|
|
373
|
+
expect(subject.instance_variable_get(:@ssl_enabled)).to be false
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
context "with explicit ssl_enabled => true and HTTPS URI" do
|
|
378
|
+
let(:avro_config) do
|
|
379
|
+
{
|
|
380
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
381
|
+
'ssl_enabled' => false
|
|
382
|
+
}
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
it "requires ssl_enabled => true" do
|
|
386
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
387
|
+
LogStash::ConfigurationError,
|
|
388
|
+
/Secured https:\/\/schema-registry.example.com\/schema.avsc connection requires `ssl_enabled => true`. /
|
|
389
|
+
)
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
context "SSL verification" do
|
|
395
|
+
context "with ssl_verification_mode => 'full'" do
|
|
396
|
+
let(:avro_config) do
|
|
397
|
+
{
|
|
398
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
399
|
+
'ssl_verification_mode' => 'full'
|
|
400
|
+
}
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
it "sets verification mode to full" do
|
|
404
|
+
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full')
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
it "builds SSL options with default verify mode" do
|
|
408
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
409
|
+
expect(ssl_options[:verify]).to eq(:default)
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
context "with ssl_verification_mode => 'none'" do
|
|
414
|
+
let(:avro_config) do
|
|
415
|
+
{
|
|
416
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
417
|
+
'ssl_verification_mode' => 'none'
|
|
418
|
+
}
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
it "sets verification mode to none" do
|
|
422
|
+
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('none')
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
it "builds SSL options with disable verify mode" do
|
|
426
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
427
|
+
expect(ssl_options[:verify]).to eq(:disable)
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
context "with default ssl_verification_mode" do
|
|
432
|
+
let(:avro_config) do
|
|
433
|
+
{
|
|
434
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc'
|
|
435
|
+
}
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
it "defaults to 'full'" do
|
|
439
|
+
subject.send(:validate_ssl_settings!)
|
|
440
|
+
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full')
|
|
441
|
+
end
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
context "keystore configuration" do
|
|
446
|
+
context "with ssl_keystore_path" do
|
|
447
|
+
let(:avro_config) do
|
|
448
|
+
{
|
|
449
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
450
|
+
'ssl_keystore_path' => paths[:test_path],
|
|
451
|
+
'ssl_keystore_password' => 'keystore_pass',
|
|
452
|
+
'ssl_keystore_type' => 'jks'
|
|
453
|
+
}
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
it "configures keystore options" do
|
|
457
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
458
|
+
expect(ssl_options[:keystore]).to eq(paths[:test_path])
|
|
459
|
+
expect(ssl_options[:keystore_password]).to eq('keystore_pass')
|
|
460
|
+
expect(ssl_options[:keystore_type]).to eq('jks')
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
context "with ssl_keystore_path and pkcs12 type" do
|
|
465
|
+
let(:avro_config) do
|
|
466
|
+
{
|
|
467
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
468
|
+
'ssl_keystore_path' => paths[:test_path],
|
|
469
|
+
'ssl_keystore_password' => 'keystore_pass',
|
|
470
|
+
'ssl_keystore_type' => 'pkcs12'
|
|
471
|
+
}
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
it "configures pkcs12 keystore" do
|
|
475
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
476
|
+
expect(ssl_options[:keystore_type]).to eq('pkcs12')
|
|
477
|
+
end
|
|
478
|
+
end
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
context "truststore configuration" do
|
|
482
|
+
context "with ssl_truststore_path" do
|
|
483
|
+
let(:avro_config) do
|
|
484
|
+
{
|
|
485
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
486
|
+
'ssl_truststore_path' => paths[:test_path],
|
|
487
|
+
'ssl_truststore_password' => 'truststore_pass',
|
|
488
|
+
'ssl_truststore_type' => 'jks'
|
|
489
|
+
}
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
it "configures truststore options" do
|
|
493
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
494
|
+
expect(ssl_options[:truststore]).to eq(paths[:test_path])
|
|
495
|
+
expect(ssl_options[:truststore_password]).to eq('truststore_pass')
|
|
496
|
+
expect(ssl_options[:truststore_type]).to eq('jks')
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
context "with ssl_truststore_path and pkcs12 type" do
|
|
501
|
+
let(:avro_config) do
|
|
502
|
+
{
|
|
503
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
504
|
+
'ssl_truststore_path' => paths[:test_path],
|
|
505
|
+
'ssl_truststore_password' => 'truststore_pass',
|
|
506
|
+
'ssl_truststore_type' => 'pkcs12'
|
|
507
|
+
}
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
it "configures pkcs12 truststore" do
|
|
511
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
512
|
+
expect(ssl_options[:truststore_type]).to eq('pkcs12')
|
|
513
|
+
end
|
|
514
|
+
end
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
context "CA configuration" do
|
|
518
|
+
|
|
519
|
+
context "with single CA file" do
|
|
520
|
+
let(:avro_config) do
|
|
521
|
+
{
|
|
522
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
523
|
+
'ssl_certificate_authorities' => [paths[:test_path]]
|
|
524
|
+
}
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
it "configures CA certificate" do
|
|
528
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
529
|
+
expect(ssl_options[:ca_file]).to eq(paths[:test_path])
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
context "with multiple CA files" do
|
|
534
|
+
let(:avro_config) do
|
|
535
|
+
{
|
|
536
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
537
|
+
'ssl_certificate_authorities' => [paths[:test_path], paths[:test_path]]
|
|
538
|
+
}
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
it "raises ConfigurationError for multiple CAs" do
|
|
542
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
543
|
+
LogStash::ConfigurationError,
|
|
544
|
+
/Multiple values on `ssl_certificate_authorities` are not supported/
|
|
545
|
+
)
|
|
546
|
+
end
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
context "with empty ssl_certificate_authorities" do
|
|
550
|
+
let(:avro_config) do
|
|
551
|
+
{
|
|
552
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
553
|
+
'ssl_certificate_authorities' => []
|
|
554
|
+
}
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
it "raises ConfigurationError" do
|
|
558
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
559
|
+
LogStash::ConfigurationError,
|
|
560
|
+
/`ssl_certificate_authorities` cannot be empty/
|
|
561
|
+
)
|
|
562
|
+
end
|
|
563
|
+
end
|
|
564
|
+
end
|
|
565
|
+
|
|
566
|
+
context "cipher suites" do
|
|
567
|
+
context "with specified cipher suites" do
|
|
568
|
+
let(:avro_config) do
|
|
569
|
+
{
|
|
570
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
571
|
+
'ssl_cipher_suites' => %w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]
|
|
572
|
+
}
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
it "configures cipher suites" do
|
|
576
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
577
|
+
expect(ssl_options[:cipher_suites]).to eq(%w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256])
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
context "without cipher suites" do
|
|
582
|
+
let(:avro_config) do
|
|
583
|
+
{
|
|
584
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc'
|
|
585
|
+
}
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
it "does not include cipher_suites in SSL options" do
|
|
589
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
590
|
+
expect(ssl_options).not_to have_key(:cipher_suites)
|
|
591
|
+
end
|
|
592
|
+
end
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
context "supported protocols" do
|
|
596
|
+
context "with TLSv1.2 and TLSv1.3" do
|
|
597
|
+
let(:avro_config) do
|
|
598
|
+
{
|
|
599
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
600
|
+
'ssl_supported_protocols' => %w[TLSv1.2 TLSv1.3]
|
|
601
|
+
}
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
it "configures supported protocols" do
|
|
605
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
606
|
+
expect(ssl_options[:protocols]).to eq(%w[TLSv1.2 TLSv1.3])
|
|
607
|
+
end
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
context "with only TLSv1.3" do
|
|
611
|
+
let(:avro_config) do
|
|
612
|
+
{
|
|
613
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
614
|
+
'ssl_supported_protocols' => ['TLSv1.3']
|
|
615
|
+
}
|
|
616
|
+
end
|
|
617
|
+
|
|
618
|
+
it "configures only TLSv1.3" do
|
|
619
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
620
|
+
expect(ssl_options[:protocols]).to eq(['TLSv1.3'])
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
|
|
624
|
+
context "with empty ssl_supported_protocols" do
|
|
625
|
+
let(:avro_config) do
|
|
626
|
+
{
|
|
627
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
628
|
+
'ssl_supported_protocols' => []
|
|
629
|
+
}
|
|
630
|
+
end
|
|
631
|
+
|
|
632
|
+
it "does not include protocols in SSL options" do
|
|
633
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
634
|
+
expect(ssl_options).not_to have_key(:protocols)
|
|
635
|
+
end
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
context "without ssl_supported_protocols" do
|
|
639
|
+
let(:avro_config) do
|
|
640
|
+
{
|
|
641
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc'
|
|
642
|
+
}
|
|
643
|
+
end
|
|
644
|
+
|
|
645
|
+
it "uses default (no protocols key)" do
|
|
646
|
+
ssl_options = subject.send(:build_ssl_options)
|
|
647
|
+
expect(ssl_options).not_to have_key(:protocols)
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
end
|
|
651
|
+
|
|
652
|
+
context "SSL validations" do
|
|
653
|
+
context "PEM certificate validation" do
|
|
654
|
+
context "when both ssl_certificate and ssl_keystore_path are set" do
|
|
655
|
+
let(:avro_config) do
|
|
656
|
+
{
|
|
657
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
658
|
+
'ssl_certificate' => paths[:test_path],
|
|
659
|
+
'ssl_key' => paths[:test_path],
|
|
660
|
+
'ssl_keystore_path' => paths[:test_path],
|
|
661
|
+
'ssl_keystore_password' => 'password'
|
|
662
|
+
}
|
|
663
|
+
end
|
|
664
|
+
|
|
665
|
+
it "raises ConfigurationError" do
|
|
666
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
667
|
+
LogStash::ConfigurationError,
|
|
668
|
+
/`ssl_certificate` and `ssl_keystore_path` cannot be used together/
|
|
669
|
+
)
|
|
670
|
+
end
|
|
671
|
+
end
|
|
672
|
+
|
|
673
|
+
context "when ssl_certificate is set without ssl_key" do
|
|
674
|
+
let(:avro_config) do
|
|
675
|
+
{
|
|
676
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
677
|
+
'ssl_certificate' => paths[:test_path]
|
|
678
|
+
}
|
|
679
|
+
end
|
|
680
|
+
|
|
681
|
+
it "raises ConfigurationError" do
|
|
682
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
683
|
+
LogStash::ConfigurationError,
|
|
684
|
+
/`ssl_certificate` requires `ssl_key`/
|
|
685
|
+
)
|
|
686
|
+
end
|
|
687
|
+
end
|
|
688
|
+
|
|
689
|
+
context "when ssl_key is set without ssl_certificate" do
|
|
690
|
+
let(:avro_config) do
|
|
691
|
+
{
|
|
692
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
693
|
+
'ssl_key' => paths[:test_path]
|
|
694
|
+
}
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
it "raises ConfigurationError" do
|
|
698
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
699
|
+
LogStash::ConfigurationError,
|
|
700
|
+
/`ssl_key` is not allowed unless `ssl_certificate` is specified/
|
|
701
|
+
)
|
|
702
|
+
end
|
|
703
|
+
end
|
|
704
|
+
end
|
|
705
|
+
|
|
706
|
+
context "keystore validation" do
|
|
707
|
+
context "when ssl_keystore_password is set without ssl_keystore_path" do
|
|
708
|
+
let(:avro_config) do
|
|
709
|
+
{
|
|
710
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
711
|
+
'ssl_keystore_password' => 'password'
|
|
712
|
+
}
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
it "raises ConfigurationError" do
|
|
716
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
717
|
+
LogStash::ConfigurationError,
|
|
718
|
+
/`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified/
|
|
719
|
+
)
|
|
720
|
+
end
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
context "when ssl_keystore_password is empty" do
|
|
724
|
+
let(:avro_config) do
|
|
725
|
+
{
|
|
726
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
727
|
+
'ssl_keystore_path' => paths[:test_path],
|
|
728
|
+
'ssl_keystore_password' => ''
|
|
729
|
+
}
|
|
730
|
+
end
|
|
731
|
+
|
|
732
|
+
it "raises ConfigurationError" do
|
|
733
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
734
|
+
LogStash::ConfigurationError,
|
|
735
|
+
/`ssl_keystore_password` cannot be empty/
|
|
736
|
+
)
|
|
737
|
+
end
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
context "when ssl_keystore_type is set without ssl_keystore_path" do
|
|
741
|
+
let(:avro_config) do
|
|
742
|
+
{
|
|
743
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
744
|
+
'ssl_keystore_type' => 'jks'
|
|
745
|
+
}
|
|
746
|
+
end
|
|
747
|
+
|
|
748
|
+
it "raises ConfigurationError" do
|
|
749
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
750
|
+
LogStash::ConfigurationError,
|
|
751
|
+
/`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified/
|
|
752
|
+
)
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
context "truststore validation" do
|
|
758
|
+
context "when ssl_truststore_password is set without ssl_truststore_path" do
|
|
759
|
+
let(:avro_config) do
|
|
760
|
+
{
|
|
761
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
762
|
+
'ssl_truststore_password' => 'password'
|
|
763
|
+
}
|
|
764
|
+
end
|
|
765
|
+
|
|
766
|
+
it "raises ConfigurationError" do
|
|
767
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
768
|
+
LogStash::ConfigurationError,
|
|
769
|
+
/`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified/
|
|
770
|
+
)
|
|
771
|
+
end
|
|
772
|
+
end
|
|
773
|
+
|
|
774
|
+
context "when ssl_truststore_password is empty" do
|
|
775
|
+
let(:avro_config) do
|
|
776
|
+
{
|
|
777
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
778
|
+
'ssl_truststore_path' => paths[:test_path],
|
|
779
|
+
'ssl_truststore_password' => ''
|
|
780
|
+
}
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
it "raises ConfigurationError" do
|
|
784
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
785
|
+
LogStash::ConfigurationError,
|
|
786
|
+
/`ssl_truststore_password` cannot be empty/
|
|
787
|
+
)
|
|
788
|
+
end
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
context "when both ssl_truststore_path and ssl_certificate_authorities are set" do
|
|
792
|
+
let(:avro_config) do
|
|
793
|
+
{
|
|
794
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
795
|
+
'ssl_truststore_path' => paths[:test_path],
|
|
796
|
+
'ssl_truststore_password' => 'password',
|
|
797
|
+
'ssl_certificate_authorities' => [paths[:test_path]]
|
|
798
|
+
}
|
|
799
|
+
end
|
|
800
|
+
|
|
801
|
+
it "raises ConfigurationError" do
|
|
802
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
803
|
+
LogStash::ConfigurationError,
|
|
804
|
+
/`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together/
|
|
805
|
+
)
|
|
806
|
+
end
|
|
807
|
+
end
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
context "verification mode validation" do
|
|
811
|
+
context "when ssl_truststore_path is set with verification mode none" do
|
|
812
|
+
let(:avro_config) do
|
|
813
|
+
{
|
|
814
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
815
|
+
'ssl_truststore_path' => paths[:test_path],
|
|
816
|
+
'ssl_truststore_password' => 'password',
|
|
817
|
+
'ssl_verification_mode' => 'none'
|
|
818
|
+
}
|
|
819
|
+
end
|
|
820
|
+
|
|
821
|
+
it "requires `ssl_verification_mode` => 'full'" do
|
|
822
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
823
|
+
LogStash::ConfigurationError,
|
|
824
|
+
/`ssl_truststore_path` requires `ssl_verification_mode` to be `full`/
|
|
825
|
+
)
|
|
826
|
+
end
|
|
827
|
+
end
|
|
828
|
+
|
|
829
|
+
context "when ssl_truststore_password is set with verification mode none" do
|
|
830
|
+
let(:avro_config) do
|
|
831
|
+
{
|
|
832
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
833
|
+
'ssl_truststore_password' => 'password',
|
|
834
|
+
'ssl_verification_mode' => 'none'
|
|
835
|
+
}
|
|
836
|
+
end
|
|
837
|
+
|
|
838
|
+
it "requires `ssl_verification_mode => 'full'" do
|
|
839
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
840
|
+
LogStash::ConfigurationError,
|
|
841
|
+
/`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`/
|
|
842
|
+
)
|
|
843
|
+
end
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
context "when ssl_certificate_authorities is set with verification mode none" do
|
|
847
|
+
let(:avro_config) do
|
|
848
|
+
{
|
|
849
|
+
'schema_uri' => 'https://schema-registry.example.com/schema.avsc',
|
|
850
|
+
'ssl_certificate_authorities' => [paths[:test_path]],
|
|
851
|
+
'ssl_verification_mode' => 'none'
|
|
852
|
+
}
|
|
853
|
+
end
|
|
854
|
+
|
|
855
|
+
it "requires `ssl_verification_mode => 'full'" do
|
|
856
|
+
expect { subject.send(:validate_ssl_settings!) }.to raise_error(
|
|
857
|
+
LogStash::ConfigurationError,
|
|
858
|
+
/`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`/
|
|
859
|
+
)
|
|
860
|
+
end
|
|
861
|
+
end
|
|
862
|
+
end
|
|
863
|
+
end
|
|
864
|
+
end
|
|
865
|
+
end
|
|
866
|
+
end
|