logstash-integration-kafka 11.1.0-java → 11.2.1-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c8f85bfdadbbd496495603c82ed577db4c23a168db59e6d0034549de6ebb66d1
4
- data.tar.gz: 0b3b0bc33d6e64eebcb9e757fede56f747625c864160844ed2a3c76d0b22a155
3
+ metadata.gz: fa69555ca0e7780e4f9ffb16c0a3308396bab86436009695708b1c03c11a7a96
4
+ data.tar.gz: e28f2ac5e38ba7cb1289496b3c633756a2607f2d37fee36fd19cc44b8abedf93
5
5
  SHA512:
6
- metadata.gz: '09e8814f1697c1d38d478881b35102a1e9885c23cdb0b9b2d8860c7577d4b2a40eb580b535afff46273e7139fbbaf603b0ea0de1fbeb68644bcea95b79d9e470'
7
- data.tar.gz: 1b0c7ee3ffbcb589268174753db91d0ba7272fc44c04e7ceb2e17d43f9fe2ec31ed4eee7450390adc0bada77c36fedaf14d46bf9b09372a6b55a6e76047a56e4
6
+ metadata.gz: fc63838c9baf545fffc3984e94faf73e72d8b97d7b4c04af12fc8de639ca55500b50255d1fcf7c6f684e49a2db763adccddb93a3df495301d66a8be769fae3f1
7
+ data.tar.gz: 540628833b9b3ab000046c07d5143812ce945de4788c58cb66680ff9c8e9d9ebdff8cb82042c52e0bbfa5a806b2deeb04ece1e8bc5e16b73edcf7f38c3be9901
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 11.2.1
2
+ - Fix nil exception to empty headers of record during event metadata assignment [#140](https://github.com/logstash-plugins/logstash-integration-kafka/pull/140)
3
+
4
+ ## 11.2.0
5
+ - Added TLS truststore and keystore settings specifically to access the schema registry [#137](https://github.com/logstash-plugins/logstash-integration-kafka/pull/137)
6
+
1
7
  ## 11.1.0
2
8
  - Added config `group_instance_id` to use the Kafka's consumer static membership feature [#135](https://github.com/logstash-plugins/logstash-integration-kafka/pull/135)
3
9
 
@@ -135,6 +135,12 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
135
135
  | <<plugins-{type}s-{plugin}-schema_registry_key>> |<<string,string>>|No
136
136
  | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
137
137
  | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
138
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No
139
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |<<password,password>>|No
140
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
141
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No
142
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |<<password,password>>|No
143
+ | <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
138
144
  | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
139
145
  | <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
140
146
  | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
@@ -598,6 +604,54 @@ Set the address of a forward HTTP proxy. An empty string is treated as if proxy
598
604
 
599
605
  Set the password for basic authorization to access remote Schema Registry.
600
606
 
607
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"]
608
+ ===== `schema_registry_ssl_keystore_location`
609
+
610
+ * Value type is <<path,path>>
611
+ * There is no default value for this setting.
612
+
613
+ If schema registry client authentication is required, this setting stores the keystore path.
614
+
615
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"]
616
+ ===== `schema_registry_ssl_keystore_password`
617
+
618
+ * Value type is <<password,password>>
619
+ * There is no default value for this setting.
620
+
621
+ If schema registry authentication is required, this setting stores the keystore password.
622
+
623
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"]
624
+ ===== `schema_registry_ssl_keystore_type`
625
+
626
+ * Value type is <<string,string>>
627
+ * There is no default value for this setting.
628
+
629
+ The format of the keystore file. It must be either `jks` or `PKCS12`.
630
+
631
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"]
632
+ ===== `schema_registry_ssl_truststore_location`
633
+
634
+ * Value type is <<path,path>>
635
+ * There is no default value for this setting.
636
+
637
+ The truststore path to validate the schema registry's certificate.
638
+
639
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"]
640
+ ===== `schema_registry_ssl_truststore_password`
641
+
642
+ * Value type is <<password,password>>
643
+ * There is no default value for this setting.
644
+
645
+ The schema registry truststore password.
646
+
647
+ [id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"]
648
+ ===== `schema_registry_ssl_truststore_type`
649
+
650
+ * Value type is <<string,string>>
651
+ * There is no default value for this setting.
652
+
653
+ The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`.
654
+
601
655
  [id="plugins-{type}s-{plugin}-schema_registry_url"]
602
656
  ===== `schema_registry_url`
603
657
 
@@ -373,7 +373,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
373
373
  event.set("[@metadata][kafka][timestamp]", record.timestamp)
374
374
  end
375
375
  if @metadata_mode.include?(:headers)
376
- record.headers.each do |header|
376
+ record.headers
377
+ .select{|h| header_with_value(h) }
378
+ .each do |header|
377
379
  s = String.from_java_bytes(header.value)
378
380
  s.force_encoding(Encoding::UTF_8)
379
381
  if s.valid_encoding?
@@ -460,6 +462,17 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
460
462
  set_trustore_keystore_config(props)
461
463
  set_sasl_config(props)
462
464
  end
465
+ if schema_registry_ssl_truststore_location
466
+ props.put('schema.registry.ssl.truststore.location', schema_registry_ssl_truststore_location)
467
+ props.put('schema.registry.ssl.truststore.password', schema_registry_ssl_truststore_password.value)
468
+ props.put('schema.registry.ssl.truststore.type', schema_registry_ssl_truststore_type)
469
+ end
470
+
471
+ if schema_registry_ssl_keystore_location
472
+ props.put('schema.registry.ssl.keystore.location', schema_registry_ssl_keystore_location)
473
+ props.put('schema.registry.ssl.keystore.password', schema_registry_ssl_keystore_password.value)
474
+ props.put('schema.registry.ssl.keystore.type', schema_registry_ssl_keystore_type)
475
+ end
463
476
 
464
477
  org.apache.kafka.clients.consumer.KafkaConsumer.new(props)
465
478
  rescue => e
@@ -488,4 +501,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
488
501
  end
489
502
  end
490
503
 
504
+ def header_with_value(header)
505
+ !header.nil? && !header.value.nil? && !header.key.nil?
506
+ end
507
+
491
508
  end #class LogStash::Inputs::Kafka
@@ -24,6 +24,24 @@ module LogStash module PluginMixins module Kafka
24
24
  # This option permits to define a proxy to be used to reach the schema registry service instance.
25
25
  config :schema_registry_proxy, :validate => :uri
26
26
 
27
+ # If schema registry client authentication is required, this setting stores the keystore path.
28
+ config :schema_registry_ssl_keystore_location, :validate => :string
29
+
30
+ # The keystore password.
31
+ config :schema_registry_ssl_keystore_password, :validate => :password
32
+
33
+ # The keystore type
34
+ config :schema_registry_ssl_keystore_type, :validate => ['jks', 'PKCS12'], :default => "jks"
35
+
36
+ # The JKS truststore path to validate the Schema Registry's certificate.
37
+ config :schema_registry_ssl_truststore_location, :validate => :string
38
+
39
+ # The truststore password.
40
+ config :schema_registry_ssl_truststore_password, :validate => :password
41
+
42
+ # The truststore type
43
+ config :schema_registry_ssl_truststore_type, :validate => ['jks', 'PKCS12'], :default => "jks"
44
+
27
45
  # Option to skip validating the schema registry during registration. This can be useful when using
28
46
  # certificate based auth
29
47
  config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
@@ -68,6 +86,19 @@ module LogStash module PluginMixins module Kafka
68
86
  if schema_registry_key and !schema_registry_key.empty?
69
87
  options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
70
88
  end
89
+ if schema_registry_ssl_truststore_location and !schema_registry_ssl_truststore_location.empty?
90
+ options[:ssl] = {} unless options.key?(:ssl)
91
+ options[:ssl][:truststore] = schema_registry_ssl_truststore_location unless schema_registry_ssl_truststore_location.nil?
92
+ options[:ssl][:truststore_password] = schema_registry_ssl_truststore_password.value unless schema_registry_ssl_truststore_password.nil?
93
+ options[:ssl][:truststore_type] = schema_registry_ssl_truststore_type unless schema_registry_ssl_truststore_type.nil?
94
+ end
95
+ if schema_registry_ssl_keystore_location and !schema_registry_ssl_keystore_location.empty?
96
+ options[:ssl] = {} unless options.key? :ssl
97
+ options[:ssl][:keystore] = schema_registry_ssl_keystore_location unless schema_registry_ssl_keystore_location.nil?
98
+ options[:ssl][:keystore_password] = schema_registry_ssl_keystore_password.value unless schema_registry_ssl_keystore_password.nil?
99
+ options[:ssl][:keystore_type] = schema_registry_ssl_keystore_type unless schema_registry_ssl_keystore_type.nil?
100
+ end
101
+
71
102
  client = Manticore::Client.new(options)
72
103
  begin
73
104
  response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-kafka'
3
- s.version = '11.1.0'
3
+ s.version = '11.2.1'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Integration with Kafka - input and output plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -353,10 +353,13 @@ describe "schema registry connection options" do
353
353
  end
354
354
  end
355
355
 
356
- def save_avro_schema_to_schema_registry(schema_file, subject_name)
356
+ def save_avro_schema_to_schema_registry(schema_file, subject_name, proto = 'http', port = 8081, manticore_options = {})
357
357
  raw_schema = File.readlines(schema_file).map(&:chomp).join
358
358
  raw_schema_quoted = raw_schema.gsub('"', '\"')
359
- response = Manticore.post("http://localhost:8081/subjects/#{subject_name}/versions",
359
+
360
+ client = Manticore::Client.new(manticore_options)
361
+
362
+ response = client.post("#{proto}://localhost:#{port}/subjects/#{subject_name}/versions",
360
363
  body: '{"schema": "' + raw_schema_quoted + '"}',
361
364
  headers: {"Content-Type" => "application/vnd.schemaregistry.v1+json"})
362
365
  response
@@ -378,8 +381,17 @@ def startup_schema_registry(schema_registry, auth=false)
378
381
  end
379
382
  end
380
383
 
381
- describe "Schema registry API", :integration => true do
382
- schema_registry = Manticore::Client.new
384
+ shared_examples 'it has endpoints available to' do |tls|
385
+ let(:port) { tls ? 8083 : 8081 }
386
+ let(:proto) { tls ? 'https' : 'http' }
387
+
388
+ manticore_options = {
389
+ :ssl => {
390
+ :truststore => File.join(Dir.pwd, "tls_repository/clienttruststore.jks"),
391
+ :truststore_password => "changeit"
392
+ }
393
+ }
394
+ schema_registry = Manticore::Client.new(manticore_options)
383
395
 
384
396
  before(:all) do
385
397
  startup_schema_registry(schema_registry)
@@ -391,36 +403,53 @@ describe "Schema registry API", :integration => true do
391
403
 
392
404
  context 'listing subject on clean instance' do
393
405
  it "should return an empty set" do
394
- subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
406
+ subjects = JSON.parse schema_registry.get("#{proto}://localhost:#{port}/subjects").body
395
407
  expect( subjects ).to be_empty
396
408
  end
397
409
  end
398
410
 
399
411
  context 'send a schema definition' do
400
412
  it "save the definition" do
401
- response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
413
+ response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1", proto, port, manticore_options)
402
414
  expect( response.code ).to be(200)
403
415
  delete_remote_schema(schema_registry, "schema_test_1")
404
416
  end
405
417
 
406
418
  it "delete the schema just added" do
407
- response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
419
+ response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1", proto, port, manticore_options)
408
420
  expect( response.code ).to be(200)
409
421
 
410
- expect( schema_registry.delete('http://localhost:8081/subjects/schema_test_1?permanent=false').code ).to be(200)
422
+ expect( schema_registry.delete("#{proto}://localhost:#{port}/subjects/schema_test_1?permanent=false").code ).to be(200)
411
423
  sleep(1)
412
- subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
424
+ subjects = JSON.parse schema_registry.get("#{proto}://localhost:#{port}/subjects").body
413
425
  expect( subjects ).to be_empty
414
426
  end
415
427
  end
416
428
  end
417
429
 
430
+ describe "Schema registry API", :integration => true do
431
+
432
+ context "when exposed with HTTPS" do
433
+ it_behaves_like 'it has endpoints available to', true
434
+ end
435
+
436
+ context "when exposed with plain HTTP" do
437
+ it_behaves_like 'it has endpoints available to', false
438
+ end
439
+ end
440
+
418
441
  def shutdown_schema_registry
419
442
  system('./stop_schema_registry.sh')
420
443
  end
421
444
 
422
445
  describe "Deserializing with the schema registry", :integration => true do
423
- schema_registry = Manticore::Client.new
446
+ manticore_options = {
447
+ :ssl => {
448
+ :truststore => File.join(Dir.pwd, "tls_repository/clienttruststore.jks"),
449
+ :truststore_password => "changeit"
450
+ }
451
+ }
452
+ schema_registry = Manticore::Client.new(manticore_options)
424
453
 
425
454
  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
426
455
 
@@ -519,28 +548,57 @@ describe "Deserializing with the schema registry", :integration => true do
519
548
  end
520
549
  end
521
550
 
522
- context 'with an unauthed schema registry' do
551
+ shared_examples 'with an unauthed schema registry' do |tls|
552
+ let(:port) { tls ? 8083 : 8081 }
553
+ let(:proto) { tls ? 'https' : 'http' }
554
+
523
555
  let(:auth) { false }
524
556
  let(:avro_topic_name) { "topic_avro" }
525
- let(:subject_url) { "http://localhost:8081/subjects" }
526
- let(:plain_config) { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
557
+ let(:subject_url) { "#{proto}://localhost:#{port}/subjects" }
558
+ let(:plain_config) { base_config.merge!({
559
+ 'schema_registry_url' => "#{proto}://localhost:#{port}",
560
+ 'schema_registry_ssl_truststore_location' => File.join(Dir.pwd, "tls_repository/clienttruststore.jks"),
561
+ 'schema_registry_ssl_truststore_password' => 'changeit',
562
+ }) }
527
563
 
528
564
  it_behaves_like 'it reads from a topic using a schema registry', false
529
565
  end
530
566
 
531
- context 'with an authed schema registry' do
567
+ context 'with an unauthed schema registry' do
568
+ context "accessed through HTTPS" do
569
+ it_behaves_like 'with an unauthed schema registry', true
570
+ end
571
+
572
+ context "accessed through HTTPS" do
573
+ it_behaves_like 'with an unauthed schema registry', false
574
+ end
575
+ end
576
+
577
+ shared_examples 'with an authed schema registry' do |tls|
578
+ let(:port) { tls ? 8083 : 8081 }
579
+ let(:proto) { tls ? 'https' : 'http' }
532
580
  let(:auth) { true }
533
581
  let(:user) { "barney" }
534
582
  let(:password) { "changeme" }
535
583
  let(:avro_topic_name) { "topic_avro_auth" }
536
- let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
584
+ let(:subject_url) { "#{proto}://#{user}:#{password}@localhost:#{port}/subjects" }
585
+ let(:tls_base_config) do
586
+ if tls
587
+ base_config.merge({
588
+ 'schema_registry_ssl_truststore_location' => ::File.join(Dir.pwd, "tls_repository/clienttruststore.jks"),
589
+ 'schema_registry_ssl_truststore_password' => 'changeit',
590
+ })
591
+ else
592
+ base_config
593
+ end
594
+ end
537
595
 
538
596
  context 'using schema_registry_key' do
539
597
  let(:plain_config) do
540
- base_config.merge!({
541
- 'schema_registry_url' => "http://localhost:8081",
598
+ tls_base_config.merge!({
599
+ 'schema_registry_url' => "#{proto}://localhost:#{port}",
542
600
  'schema_registry_key' => user,
543
- 'schema_registry_secret' => password
601
+ 'schema_registry_secret' => password,
544
602
  })
545
603
  end
546
604
 
@@ -549,12 +607,22 @@ describe "Deserializing with the schema registry", :integration => true do
549
607
 
550
608
  context 'using schema_registry_url' do
551
609
  let(:plain_config) do
552
- base_config.merge!({
553
- 'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
610
+ tls_base_config.merge!({
611
+ 'schema_registry_url' => "#{proto}://#{user}:#{password}@localhost:#{port}",
554
612
  })
555
613
  end
556
614
 
557
615
  it_behaves_like 'it reads from a topic using a schema registry', true
558
616
  end
559
617
  end
618
+
619
+ context 'with an authed schema registry' do
620
+ context "accessed through HTTPS" do
621
+ it_behaves_like 'with an authed schema registry', true
622
+ end
623
+
624
+ context "accessed through HTTPS" do
625
+ it_behaves_like 'with an authed schema registry', false
626
+ end
627
+ end
560
628
  end
@@ -287,6 +287,19 @@ describe LogStash::Inputs::Kafka do
287
287
  subject.register
288
288
  expect(subject.metadata_mode).to include(:record_props)
289
289
  end
290
+
291
+ context "guards against nil header" do
292
+ let(:header) { double(:value => nil, :key => "k") }
293
+ let(:headers) { [ header ] }
294
+ let(:record) { double(:headers => headers, :topic => "topic", :partition => 0,
295
+ :offset => 123456789, :key => "someId", :timestamp => nil ) }
296
+
297
+ it "does not raise error when key is nil" do
298
+ subject.register
299
+ evt = LogStash::Event.new('message' => 'Hello')
300
+ expect { subject.maybe_set_metadata(evt, record) }.not_to raise_error
301
+ end
302
+ end
290
303
  end
291
304
 
292
305
  context 'with client_rack' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.1.0
4
+ version: 11.2.1
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-01-25 00:00:00.000000000 Z
11
+ date: 2023-02-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement