logstash-codec-avro 3.4.1-java → 3.5.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/docs/index.asciidoc +222 -0
- data/lib/logstash/codecs/avro.rb +248 -5
- data/logstash-codec-avro.gemspec +2 -1
- data/spec/integration/avro_integration_spec.rb +431 -0
- data/spec/integration/fixtures/jaas.config +5 -0
- data/spec/integration/fixtures/pwd +2 -0
- data/spec/integration/fixtures/trust-store_stub.jks +0 -0
- data/spec/integration/kafka_test_setup.sh +85 -0
- data/spec/integration/kafka_test_teardown.sh +16 -0
- data/spec/integration/setup_keystore_and_truststore.sh +17 -0
- data/spec/integration/start_auth_schema_registry.sh +8 -0
- data/spec/integration/start_schema_registry.sh +5 -0
- data/spec/integration/start_schema_registry_mutual.sh +5 -0
- data/spec/integration/stop_schema_registry.sh +6 -0
- data/spec/unit/avro_spec.rb +866 -0
- metadata +61 -22
- data/spec/codecs/avro_spec.rb +0 -203
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require 'logstash/devutils/rspec/spec_helper'
|
|
3
|
+
require 'logstash/codecs/avro'
|
|
4
|
+
require 'logstash/event'
|
|
5
|
+
require 'avro'
|
|
6
|
+
require 'base64'
|
|
7
|
+
require 'manticore'
|
|
8
|
+
|
|
9
|
+
describe "Avro Codec Integration Tests", :integration => true do
|
|
10
|
+
INTEGRATION_DIR = File.expand_path('../', __FILE__)
|
|
11
|
+
|
|
12
|
+
let(:test_schema) do
|
|
13
|
+
{
|
|
14
|
+
"type" => "record",
|
|
15
|
+
"name" => "TestRecord",
|
|
16
|
+
"namespace" => "com.example",
|
|
17
|
+
"fields" => [
|
|
18
|
+
{ "name" => "message", "type" => "string" },
|
|
19
|
+
{ "name" => "timestamp", "type" => "long" }
|
|
20
|
+
]
|
|
21
|
+
}
|
|
22
|
+
end
|
|
23
|
+
let(:test_schema_json) { test_schema.to_json }
|
|
24
|
+
let(:test_event_data) do
|
|
25
|
+
{
|
|
26
|
+
"message" => "test message",
|
|
27
|
+
"timestamp" => Time.now.to_i
|
|
28
|
+
}
|
|
29
|
+
end
|
|
30
|
+
let(:config) { {} }
|
|
31
|
+
let(:codec) { LogStash::Codecs::Avro.new(config).tap { |c| c.register } }
|
|
32
|
+
|
|
33
|
+
def run_integration_script(script_name)
|
|
34
|
+
Dir.chdir(INTEGRATION_DIR) do
|
|
35
|
+
result = system("./#{script_name}")
|
|
36
|
+
puts "Script #{script_name} #{result ? 'succeeded' : 'failed'}"
|
|
37
|
+
result
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def encode_avro_data(schema_json, data)
|
|
42
|
+
schema = Avro::Schema.parse(schema_json)
|
|
43
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
|
44
|
+
buffer = StringIO.new
|
|
45
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
|
46
|
+
dw.write(data, encoder)
|
|
47
|
+
Base64.strict_encode64(buffer.string)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def decode_with_codec(codec, encoded_data)
|
|
51
|
+
events = []
|
|
52
|
+
codec.decode(encoded_data) do |event|
|
|
53
|
+
events << event
|
|
54
|
+
end
|
|
55
|
+
events
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def expect_decoded_event_matches(events, expected_data)
|
|
59
|
+
expect(events.size).to eq(1)
|
|
60
|
+
expect(events.first.get("message")).to eq(expected_data["message"])
|
|
61
|
+
expect(events.first.get("timestamp")).to eq(expected_data["timestamp"])
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def create_manticore_client(username: nil, password: nil, ssl_options: {})
|
|
65
|
+
client_options = {}
|
|
66
|
+
if username && password
|
|
67
|
+
client_options[:auth] = { user: username, password: password }
|
|
68
|
+
end
|
|
69
|
+
client_options[:ssl] = ssl_options unless ssl_options.empty?
|
|
70
|
+
|
|
71
|
+
Manticore::Client.new(client_options)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {})
|
|
75
|
+
puts "Waiting for Schema Registry at #{url}..."
|
|
76
|
+
client = create_manticore_client(username: username, password: password, ssl_options: ssl_options)
|
|
77
|
+
|
|
78
|
+
Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
|
|
79
|
+
response = client.get(url).call
|
|
80
|
+
expect(response.code).to eq(200)
|
|
81
|
+
end
|
|
82
|
+
client.close
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def register_schema(base_url, subject, schema_json, username: nil, password: nil, ssl_options: {})
|
|
86
|
+
client = create_manticore_client(username: username, password: password, ssl_options: ssl_options)
|
|
87
|
+
response = client.post("#{base_url}/subjects/#{subject}/versions",
|
|
88
|
+
headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" },
|
|
89
|
+
body: { schema: schema_json }.to_json
|
|
90
|
+
).call
|
|
91
|
+
expect(response.code).to eq(200)
|
|
92
|
+
client.close
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
context "Schema Registry without authentication" do
|
|
96
|
+
let(:schema_registry_url) { "http://localhost:8081" }
|
|
97
|
+
|
|
98
|
+
before(:all) do
|
|
99
|
+
run_integration_script("start_schema_registry.sh")
|
|
100
|
+
wait_for_schema_registry("http://localhost:8081")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
after(:all) do
|
|
104
|
+
run_integration_script("stop_schema_registry.sh")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
context "fetching schema via HTTP" do
|
|
108
|
+
let(:schema_subject) { "test-no-auth-#{Time.now.to_i}" }
|
|
109
|
+
let(:full_schema_url) do
|
|
110
|
+
url = "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest"
|
|
111
|
+
puts "Constructed schema URL: #{url}"
|
|
112
|
+
url
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
let(:config) { super().merge({ 'schema_uri' => full_schema_url }) }
|
|
116
|
+
|
|
117
|
+
before do
|
|
118
|
+
register_schema(schema_registry_url, schema_subject, test_schema_json)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it "fetches and decodes schema from Schema Registry" do
|
|
122
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
123
|
+
events = decode_with_codec(codec, encoded_data)
|
|
124
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
it "encodes data using schema from schema registry" do
|
|
128
|
+
event = LogStash::Event.new(test_event_data)
|
|
129
|
+
encoded_data = nil
|
|
130
|
+
|
|
131
|
+
codec.on_event do |e, data|
|
|
132
|
+
encoded_data = data
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
codec.encode(event)
|
|
136
|
+
|
|
137
|
+
expect(encoded_data).not_to be_nil
|
|
138
|
+
|
|
139
|
+
events = decode_with_codec(codec, encoded_data)
|
|
140
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
context "Schema Registry with authentication" do
|
|
146
|
+
let(:schema_registry_url) { "http://localhost:8081" }
|
|
147
|
+
let(:username) { "barney" }
|
|
148
|
+
let(:password) { "changeme" }
|
|
149
|
+
|
|
150
|
+
before(:all) do
|
|
151
|
+
run_integration_script("stop_schema_registry.sh")
|
|
152
|
+
run_integration_script("start_auth_schema_registry.sh")
|
|
153
|
+
wait_for_schema_registry("http://localhost:8081", username: "barney", password: "changeme")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
after(:all) do
|
|
157
|
+
run_integration_script("stop_schema_registry.sh")
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
context "with valid credentials" do
|
|
161
|
+
let(:schema_subject) { "test-auth-#{Time.now.to_i}" }
|
|
162
|
+
let(:full_schema_url) { "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" }
|
|
163
|
+
let(:config) { super().merge({ 'schema_uri' => full_schema_url, 'username' => username, 'password' => password }) }
|
|
164
|
+
|
|
165
|
+
before do
|
|
166
|
+
register_schema(schema_registry_url, schema_subject, test_schema_json,
|
|
167
|
+
username: username, password: password)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
it "fetches schema with valid credentials" do
|
|
171
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
172
|
+
events = decode_with_codec(codec, encoded_data)
|
|
173
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
it "encodes data with authentication" do
|
|
177
|
+
event = LogStash::Event.new(test_event_data)
|
|
178
|
+
encoded_data = nil
|
|
179
|
+
|
|
180
|
+
codec.on_event do |e, data|
|
|
181
|
+
encoded_data = data
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
codec.encode(event)
|
|
185
|
+
expect(encoded_data).not_to be_nil
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
context "with invalid credentials" do
|
|
190
|
+
let(:schema_subject) { "test-invalid-auth-#{Time.now.to_i}" }
|
|
191
|
+
let(:full_schema_url) { "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" }
|
|
192
|
+
|
|
193
|
+
before do
|
|
194
|
+
register_schema(schema_registry_url, schema_subject, test_schema_json,
|
|
195
|
+
username: username, password: password)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
it "fails with invalid credentials" do
|
|
199
|
+
expect {
|
|
200
|
+
invalid_config = { 'schema_uri' => full_schema_url, 'username' => 'invalid', 'password' => 'wrong' }
|
|
201
|
+
codec = LogStash::Codecs::Avro.new(invalid_config)
|
|
202
|
+
codec.register
|
|
203
|
+
}.to raise_error { |error|
|
|
204
|
+
expect(error.message).to include("Unauthorized")
|
|
205
|
+
}
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
context "Schema Registry with truststore configuration" do
|
|
211
|
+
let(:schema_registry_https_url) { "https://localhost:8083" }
|
|
212
|
+
let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") }
|
|
213
|
+
let(:truststore_password) { "changeit" }
|
|
214
|
+
let(:ca_cert_path) { File.join(INTEGRATION_DIR, "tls_repository", "schema_reg_certificate.pem") }
|
|
215
|
+
|
|
216
|
+
before(:all) do
|
|
217
|
+
# Ensure non-auth registry is running (it includes HTTPS on 8083)
|
|
218
|
+
run_integration_script("stop_schema_registry.sh")
|
|
219
|
+
run_integration_script("start_schema_registry.sh")
|
|
220
|
+
|
|
221
|
+
ssl_options = {
|
|
222
|
+
truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"),
|
|
223
|
+
truststore_password: "changeit",
|
|
224
|
+
truststore_type: "jks",
|
|
225
|
+
verify: :default
|
|
226
|
+
}
|
|
227
|
+
wait_for_schema_registry("https://localhost:8083", ssl_options: ssl_options)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
after(:all) do
|
|
231
|
+
run_integration_script("stop_schema_registry.sh")
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
context "with truststore configuration" do
|
|
235
|
+
let(:schema_subject) { "test-ssl-truststore-#{Time.now.to_i}" }
|
|
236
|
+
let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" }
|
|
237
|
+
let(:config) do
|
|
238
|
+
super().merge({
|
|
239
|
+
'schema_uri' => full_schema_url,
|
|
240
|
+
'ssl_enabled' => true,
|
|
241
|
+
'ssl_truststore_path' => truststore_path,
|
|
242
|
+
'ssl_truststore_password' => truststore_password,
|
|
243
|
+
'ssl_truststore_type' => 'jks',
|
|
244
|
+
'ssl_verification_mode' => 'full'
|
|
245
|
+
})
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
before do
|
|
249
|
+
ssl_options = {
|
|
250
|
+
truststore: truststore_path,
|
|
251
|
+
truststore_password: truststore_password,
|
|
252
|
+
truststore_type: "jks",
|
|
253
|
+
verify: :default
|
|
254
|
+
}
|
|
255
|
+
register_schema(schema_registry_https_url, schema_subject, test_schema_json,
|
|
256
|
+
ssl_options: ssl_options)
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
it "fetches schema using truststore" do
|
|
260
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
261
|
+
events = decode_with_codec(codec, encoded_data)
|
|
262
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
context "with CA certificate configuration" do
|
|
267
|
+
let(:schema_subject) { "test-ssl-ca-#{Time.now.to_i}" }
|
|
268
|
+
let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" }
|
|
269
|
+
let(:config) do
|
|
270
|
+
super().merge({
|
|
271
|
+
'schema_uri' => full_schema_url,
|
|
272
|
+
'ssl_enabled' => true,
|
|
273
|
+
'ssl_certificate_authorities' => [ca_cert_path],
|
|
274
|
+
'ssl_verification_mode' => 'full'
|
|
275
|
+
})
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
before do
|
|
279
|
+
ssl_options = { ca_file: ca_cert_path, verify: :default }
|
|
280
|
+
register_schema(schema_registry_https_url, schema_subject, test_schema_json,
|
|
281
|
+
ssl_options: ssl_options)
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
it "fetches schema using CA certificate" do
|
|
285
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
286
|
+
events = decode_with_codec(codec, encoded_data)
|
|
287
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
context "Schema Registry with authentication and SSL" do
|
|
293
|
+
let(:schema_registry_https_url) { "https://localhost:8083" }
|
|
294
|
+
let(:username) { "barney" }
|
|
295
|
+
let(:password) { "changeme" }
|
|
296
|
+
let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") }
|
|
297
|
+
let(:truststore_password) { "changeit" }
|
|
298
|
+
|
|
299
|
+
before(:all) do
|
|
300
|
+
# Start authenticated registry (includes HTTPS)
|
|
301
|
+
run_integration_script("stop_schema_registry.sh")
|
|
302
|
+
run_integration_script("start_auth_schema_registry.sh")
|
|
303
|
+
|
|
304
|
+
ssl_options = {
|
|
305
|
+
truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"),
|
|
306
|
+
truststore_password: "changeit",
|
|
307
|
+
truststore_type: "jks",
|
|
308
|
+
verify: :default
|
|
309
|
+
}
|
|
310
|
+
wait_for_schema_registry("https://localhost:8083", username: "barney", password: "changeme", ssl_options: ssl_options)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
after(:all) do
|
|
314
|
+
run_integration_script("stop_schema_registry.sh")
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
context "with valid credentials and truststore" do
|
|
318
|
+
let(:schema_subject) { "test-auth-ssl-#{Time.now.to_i}" }
|
|
319
|
+
let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" }
|
|
320
|
+
let(:config) do
|
|
321
|
+
super().merge({
|
|
322
|
+
'schema_uri' => full_schema_url,
|
|
323
|
+
'username' => username,
|
|
324
|
+
'password' => password,
|
|
325
|
+
'ssl_enabled' => true,
|
|
326
|
+
'ssl_truststore_path' => truststore_path,
|
|
327
|
+
'ssl_truststore_password' => truststore_password,
|
|
328
|
+
'ssl_truststore_type' => 'jks',
|
|
329
|
+
'ssl_verification_mode' => 'full'
|
|
330
|
+
})
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
before do
|
|
334
|
+
ssl_options = {
|
|
335
|
+
truststore: truststore_path,
|
|
336
|
+
truststore_password: truststore_password,
|
|
337
|
+
truststore_type: "jks",
|
|
338
|
+
verify: :default
|
|
339
|
+
}
|
|
340
|
+
register_schema(schema_registry_https_url, schema_subject, test_schema_json,
|
|
341
|
+
username: username, password: password,
|
|
342
|
+
ssl_options: ssl_options)
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
it "fetches schema with both authentication and SSL" do
|
|
346
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
347
|
+
events = decode_with_codec(codec, encoded_data)
|
|
348
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
it "encodes data with authentication and SSL" do
|
|
352
|
+
event = LogStash::Event.new(test_event_data)
|
|
353
|
+
encoded_data = nil
|
|
354
|
+
|
|
355
|
+
codec.on_event do |e, data|
|
|
356
|
+
encoded_data = data
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
codec.encode(event)
|
|
360
|
+
expect(encoded_data).not_to be_nil
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
context "Schema Registry with keystore configuration (mutual TLS)" do
|
|
366
|
+
let(:schema_registry_https_url) { "https://localhost:8083" }
|
|
367
|
+
|
|
368
|
+
let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") }
|
|
369
|
+
let(:truststore_password) { "changeit" }
|
|
370
|
+
let(:keystore_path) { File.join(INTEGRATION_DIR, "tls_repository", "schema_reg.jks") }
|
|
371
|
+
let(:keystore_password) { "changeit" }
|
|
372
|
+
|
|
373
|
+
before(:all) do
|
|
374
|
+
run_integration_script("stop_schema_registry.sh")
|
|
375
|
+
run_integration_script("start_schema_registry_mutual.sh")
|
|
376
|
+
|
|
377
|
+
ssl_options = {
|
|
378
|
+
keystore: File.join(INTEGRATION_DIR, "tls_repository", "schema_reg.jks"),
|
|
379
|
+
keystore_password: "changeit",
|
|
380
|
+
keystore_type: "jks",
|
|
381
|
+
truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"),
|
|
382
|
+
truststore_password: "changeit",
|
|
383
|
+
truststore_type: "jks"
|
|
384
|
+
}
|
|
385
|
+
wait_for_schema_registry("https://localhost:8083", ssl_options: ssl_options)
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
after(:all) do
|
|
389
|
+
run_integration_script("stop_schema_registry.sh")
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
context "with keystore and truststore" do
|
|
393
|
+
let(:schema_subject) { "test-mutual-tls-#{Time.now.to_i}" }
|
|
394
|
+
let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" }
|
|
395
|
+
|
|
396
|
+
let(:config) do
|
|
397
|
+
super().merge({
|
|
398
|
+
'schema_uri' => full_schema_url,
|
|
399
|
+
'ssl_enabled' => true,
|
|
400
|
+
'ssl_keystore_path' => keystore_path,
|
|
401
|
+
'ssl_keystore_password' => keystore_password,
|
|
402
|
+
'ssl_keystore_type' => 'jks',
|
|
403
|
+
'ssl_truststore_path' => truststore_path,
|
|
404
|
+
'ssl_truststore_password' => truststore_password,
|
|
405
|
+
'ssl_truststore_type' => 'jks',
|
|
406
|
+
'ssl_verification_mode' => 'full'
|
|
407
|
+
})
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
before do
|
|
411
|
+
ssl_options = {
|
|
412
|
+
keystore: keystore_path,
|
|
413
|
+
keystore_password: keystore_password,
|
|
414
|
+
keystore_type: "jks",
|
|
415
|
+
truststore: truststore_path,
|
|
416
|
+
truststore_password: truststore_password,
|
|
417
|
+
truststore_type: "jks",
|
|
418
|
+
verify: :default
|
|
419
|
+
}
|
|
420
|
+
register_schema(schema_registry_https_url, schema_subject, test_schema_json,
|
|
421
|
+
ssl_options: ssl_options)
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
it "fetches schema" do
|
|
425
|
+
encoded_data = encode_avro_data(test_schema_json, test_event_data)
|
|
426
|
+
events = decode_with_codec(codec, encoded_data)
|
|
427
|
+
expect_decoded_event_matches(events, test_event_data)
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
end
|
|
File without changes
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Setup Kafka and create test topics
|
|
3
|
+
|
|
4
|
+
set -ex
|
|
5
|
+
# check if KAFKA_VERSION env var is set
|
|
6
|
+
if [ -n "${KAFKA_VERSION+1}" ]; then
|
|
7
|
+
echo "KAFKA_VERSION is $KAFKA_VERSION"
|
|
8
|
+
else
|
|
9
|
+
KAFKA_VERSION=4.1.0
|
|
10
|
+
fi
|
|
11
|
+
|
|
12
|
+
KAFKA_MAJOR_VERSION="${KAFKA_VERSION%%.*}"
|
|
13
|
+
|
|
14
|
+
export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
|
|
15
|
+
|
|
16
|
+
rm -rf build
|
|
17
|
+
mkdir build
|
|
18
|
+
|
|
19
|
+
echo "Setup Kafka version $KAFKA_VERSION"
|
|
20
|
+
if [ ! -e "kafka_2.13-$KAFKA_VERSION.tgz" ]; then
|
|
21
|
+
echo "Kafka not present locally, downloading"
|
|
22
|
+
curl -s -o "kafka_2.13-$KAFKA_VERSION.tgz" "https://downloads.apache.org/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
|
|
23
|
+
fi
|
|
24
|
+
cp "kafka_2.13-$KAFKA_VERSION.tgz" "build/kafka.tgz"
|
|
25
|
+
mkdir "build/kafka" && tar xzf "build/kafka.tgz" -C "build/kafka" --strip-components 1
|
|
26
|
+
|
|
27
|
+
echo "Use KRaft for Kafka version $KAFKA_VERSION"
|
|
28
|
+
echo "log.dirs=${PWD}/build/kafka-logs" >> build/kafka/config/server.properties
|
|
29
|
+
|
|
30
|
+
build/kafka/bin/kafka-storage.sh format \
|
|
31
|
+
--cluster-id $(build/kafka/bin/kafka-storage.sh random-uuid) \
|
|
32
|
+
--config build/kafka/config/server.properties \
|
|
33
|
+
--ignore-formatted \
|
|
34
|
+
--standalone
|
|
35
|
+
|
|
36
|
+
echo "Starting Kafka broker"
|
|
37
|
+
build/kafka/bin/kafka-server-start.sh -daemon "build/kafka/config/server.properties" --override advertised.host.name=127.0.0.1 --override log.dirs="${PWD}/build/kafka-logs"
|
|
38
|
+
sleep 10
|
|
39
|
+
|
|
40
|
+
echo "Setup Confluent Platform"
|
|
41
|
+
# check if CONFLUENT_VERSION env var is set
|
|
42
|
+
if [ -n "${CONFLUENT_VERSION+1}" ]; then
|
|
43
|
+
echo "CONFLUENT_VERSION is $CONFLUENT_VERSION"
|
|
44
|
+
else
|
|
45
|
+
CONFLUENT_VERSION=8.0.0
|
|
46
|
+
fi
|
|
47
|
+
if [ ! -e "confluent-community-$CONFLUENT_VERSION.tar.gz" ]; then
|
|
48
|
+
echo "Confluent Platform not present locally, downloading"
|
|
49
|
+
CONFLUENT_MINOR=$(echo "$CONFLUENT_VERSION" | sed -n 's/^\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p')
|
|
50
|
+
echo "CONFLUENT_MINOR is $CONFLUENT_MINOR"
|
|
51
|
+
curl -s -o "confluent-community-$CONFLUENT_VERSION.tar.gz" "http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION.tar.gz"
|
|
52
|
+
fi
|
|
53
|
+
echo "Extracting confluent-community-$CONFLUENT_VERSION.tar.gz to build"
|
|
54
|
+
mkdir "build/confluent_platform" && tar xzf "confluent-community-$CONFLUENT_VERSION.tar.gz" -C "build/confluent_platform" --strip-components 1
|
|
55
|
+
|
|
56
|
+
echo "Configuring TLS on Schema registry"
|
|
57
|
+
rm -Rf tls_repository
|
|
58
|
+
mkdir tls_repository
|
|
59
|
+
./setup_keystore_and_truststore.sh
|
|
60
|
+
# configure schema-registry to handle https on 8083 port
|
|
61
|
+
if [[ "$OSTYPE" == "darwin"* ]]; then
|
|
62
|
+
sed -i '' 's/http:\/\/0.0.0.0:8081/http:\/\/0.0.0.0:8081, https:\/\/0.0.0.0:8083/g' "build/confluent_platform/etc/schema-registry/schema-registry.properties"
|
|
63
|
+
else
|
|
64
|
+
sed -i 's/http:\/\/0.0.0.0:8081/http:\/\/0.0.0.0:8081, https:\/\/0.0.0.0:8083/g' "build/confluent_platform/etc/schema-registry/schema-registry.properties"
|
|
65
|
+
fi
|
|
66
|
+
echo "ssl.keystore.location=`pwd`/tls_repository/schema_reg.jks" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties"
|
|
67
|
+
echo "ssl.keystore.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties"
|
|
68
|
+
echo "ssl.key.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties"
|
|
69
|
+
|
|
70
|
+
cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties"
|
|
71
|
+
echo "ssl.truststore.location=`pwd`/tls_repository/clienttruststore.jks" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties"
|
|
72
|
+
echo "ssl.truststore.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties"
|
|
73
|
+
echo "confluent.http.server.ssl.client.authentication=REQUIRED" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties"
|
|
74
|
+
|
|
75
|
+
cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties"
|
|
76
|
+
echo "authentication.method=BASIC" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties"
|
|
77
|
+
echo "authentication.roles=admin,developer,user" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties"
|
|
78
|
+
echo "authentication.realm=SchemaRegistry-Props" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties"
|
|
79
|
+
cp fixtures/jaas.config "build/confluent_platform/etc/schema-registry"
|
|
80
|
+
|
|
81
|
+
echo "Setting up a test topic"
|
|
82
|
+
build/kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic logstash_integration_topic_plain --bootstrap-server localhost:9092
|
|
83
|
+
|
|
84
|
+
cp fixtures/pwd "build/confluent_platform/etc/schema-registry"
|
|
85
|
+
echo "Setup complete, running specs"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -ex
|
|
3
|
+
|
|
4
|
+
echo "Unregistering test topics"
|
|
5
|
+
#build/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --delete --topic 'topic_avro.*' 2>&1
|
|
6
|
+
|
|
7
|
+
echo "Stopping Kafka broker"
|
|
8
|
+
build/kafka/bin/kafka-server-stop.sh
|
|
9
|
+
|
|
10
|
+
if [ -f "build/kafka/bin/zookeeper-server-stop.sh" ]; then
|
|
11
|
+
echo "Stopping ZooKeeper"
|
|
12
|
+
build/kafka/bin/zookeeper-server-stop.sh
|
|
13
|
+
fi
|
|
14
|
+
|
|
15
|
+
echo "Clean TLS folder"
|
|
16
|
+
rm -Rf tls_repository
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Setup Schema Registry keystore and Kafka's schema registry client's truststore
|
|
3
|
+
set -ex
|
|
4
|
+
|
|
5
|
+
echo "Generating schema registry key store"
|
|
6
|
+
keytool -genkey -alias schema_reg -keyalg RSA -keystore tls_repository/schema_reg.jks -keypass changeit -storepass changeit -validity 365 -keysize 2048 -dname "CN=localhost, OU=John Doe, O=Acme Inc, L=Unknown, ST=Unknown, C=IT"
|
|
7
|
+
|
|
8
|
+
echo "Exporting schema registry certificate"
|
|
9
|
+
keytool -exportcert -rfc -keystore tls_repository/schema_reg.jks -storepass changeit -alias schema_reg -file tls_repository/schema_reg_certificate.pem
|
|
10
|
+
|
|
11
|
+
echo "Creating client's truststore and importing schema registry's certificate"
|
|
12
|
+
keytool -import -trustcacerts -file tls_repository/schema_reg_certificate.pem -keypass changeit -storepass changeit -keystore tls_repository/clienttruststore.jks -noprompt
|
|
13
|
+
|
|
14
|
+
# make files read only
|
|
15
|
+
chmod 444 tls_repository/schema_reg.jks
|
|
16
|
+
chmod 444 tls_repository/schema_reg_certificate.pem
|
|
17
|
+
chmod 444 tls_repository/clienttruststore.jks
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -ex
|
|
3
|
+
|
|
4
|
+
echo "Starting authed SchemaRegistry"
|
|
5
|
+
SCHEMA_REGISTRY_OPTS="-Djava.security.auth.login.config=build/confluent_platform/etc/schema-registry/jaas.config" \
|
|
6
|
+
build/confluent_platform/bin/schema-registry-start \
|
|
7
|
+
build/confluent_platform/etc/schema-registry/authed-schema-registry.properties \
|
|
8
|
+
> /dev/null 2>&1 &
|