ruby-kafka 0.5.4 → 0.5.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -1
- data/README.md +0 -1
- data/lib/kafka/broker_uri.rb +41 -0
- data/lib/kafka/client.rb +10 -50
- data/lib/kafka/cluster.rb +5 -0
- data/lib/kafka/ssl_context.rb +39 -0
- data/lib/kafka/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63ec54e4024c83bb5a73583700391db7b89fed005bb4b196cf912424351bba0e
|
4
|
+
data.tar.gz: d567d51c5434f6ef034b769f78cc03286d91e875aa5ea9058623cd2e755cf28c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3f9f8269075c30eb60a3ee7afcca5c1fe8eab9aa6c5cf51c5512dc6b564299a5366d33bc97d08b61a17ecc4a299e066ecb48f86938c93643324a8083289ed3af
|
7
|
+
data.tar.gz: 92767b83a152bef74992d1b4dc4b1818bdce8715a4e1e32c53671eaf6ee8d28d8e76d4800bf141c8cdd6cc44ef51aef78335464dbe504f8f72245ec8fc964a9a
|
data/CHANGELOG.md
CHANGED
@@ -4,9 +4,13 @@ Changes and additions to the library will be listed here.
|
|
4
4
|
|
5
5
|
## Unreleased
|
6
6
|
|
7
|
+
## v0.5.5
|
8
|
+
|
9
|
+
- Support PLAINTEXT and SSL URI schemes (#550).
|
10
|
+
|
7
11
|
## v0.5.4
|
8
12
|
|
9
|
-
- Add support for config entries in the topic creation API.
|
13
|
+
- Add support for config entries in the topic creation API (#540).
|
10
14
|
- Don't fail on retry when the cluster is secured (#545).
|
11
15
|
|
12
16
|
## v0.5.3
|
data/README.md
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
A Ruby client library for [Apache Kafka](http://kafka.apache.org/), a distributed log and message bus. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
|
4
4
|
|
5
|
-
Although parts of this library work with Kafka 0.8 – specifically, the Producer API – it's being tested and developed against Kafka 0.9. The Consumer API is Kafka 0.9+ only.
|
6
5
|
|
7
6
|
## Table of Contents
|
8
7
|
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "uri"
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
module BrokerUri
|
5
|
+
DEFAULT_PORT = 9092
|
6
|
+
URI_SCHEMES = ["kafka", "kafka+ssl", "plaintext", "ssl"]
|
7
|
+
|
8
|
+
# Parses a Kafka broker URI string.
|
9
|
+
#
|
10
|
+
# Examples of valid strings:
|
11
|
+
# * `kafka1.something`
|
12
|
+
# * `kafka1.something:1234`
|
13
|
+
# * `kafka://kafka1.something:1234`
|
14
|
+
# * `kafka+ssl://kafka1.something:1234`
|
15
|
+
# * `plaintext://kafka1.something:1234`
|
16
|
+
#
|
17
|
+
# @param str [String] a Kafka broker URI string.
|
18
|
+
# @return [URI]
|
19
|
+
def self.parse(str)
|
20
|
+
# Make sure there's a scheme part if it's missing.
|
21
|
+
str = "kafka://" + str unless str.include?("://")
|
22
|
+
|
23
|
+
uri = URI.parse(str)
|
24
|
+
uri.port ||= DEFAULT_PORT
|
25
|
+
|
26
|
+
# Map some schemes to others.
|
27
|
+
case uri.scheme
|
28
|
+
when 'plaintext'
|
29
|
+
uri.scheme = 'kafka'
|
30
|
+
when 'ssl'
|
31
|
+
uri.scheme = 'kafka+ssl'
|
32
|
+
end
|
33
|
+
|
34
|
+
unless URI_SCHEMES.include?(uri.scheme)
|
35
|
+
raise Kafka::Error, "invalid protocol `#{uri.scheme}` in `#{str}`"
|
36
|
+
end
|
37
|
+
|
38
|
+
uri
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/kafka/client.rb
CHANGED
@@ -1,10 +1,9 @@
|
|
1
|
-
require "
|
2
|
-
require "uri"
|
3
|
-
|
1
|
+
require "kafka/ssl_context"
|
4
2
|
require "kafka/cluster"
|
5
3
|
require "kafka/producer"
|
6
4
|
require "kafka/consumer"
|
7
5
|
require "kafka/heartbeat"
|
6
|
+
require "kafka/broker_uri"
|
8
7
|
require "kafka/async_producer"
|
9
8
|
require "kafka/fetched_message"
|
10
9
|
require "kafka/fetch_operation"
|
@@ -14,8 +13,6 @@ require "kafka/sasl_authenticator"
|
|
14
13
|
|
15
14
|
module Kafka
|
16
15
|
class Client
|
17
|
-
URI_SCHEMES = ["kafka", "kafka+ssl"]
|
18
|
-
|
19
16
|
# Initializes a new Kafka client.
|
20
17
|
#
|
21
18
|
# @param seed_brokers [Array<String>, String] the list of brokers used to initialize
|
@@ -65,7 +62,13 @@ module Kafka
|
|
65
62
|
@instrumenter = Instrumenter.new(client_id: client_id)
|
66
63
|
@seed_brokers = normalize_seed_brokers(seed_brokers)
|
67
64
|
|
68
|
-
ssl_context =
|
65
|
+
ssl_context = SslContext.build(
|
66
|
+
ca_cert_file_path: ssl_ca_cert_file_path,
|
67
|
+
ca_cert: ssl_ca_cert,
|
68
|
+
client_cert: ssl_client_cert,
|
69
|
+
client_cert_key: ssl_client_cert_key,
|
70
|
+
ca_certs_from_system: ssl_ca_certs_from_system,
|
71
|
+
)
|
69
72
|
|
70
73
|
sasl_authenticator = SaslAuthenticator.new(
|
71
74
|
sasl_gssapi_principal: sasl_gssapi_principal,
|
@@ -605,55 +608,12 @@ module Kafka
|
|
605
608
|
)
|
606
609
|
end
|
607
610
|
|
608
|
-
def build_ssl_context(ca_cert_file_path, ca_cert, client_cert, client_cert_key, ssl_ca_certs_from_system)
|
609
|
-
return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || ssl_ca_certs_from_system
|
610
|
-
|
611
|
-
ssl_context = OpenSSL::SSL::SSLContext.new
|
612
|
-
|
613
|
-
if client_cert && client_cert_key
|
614
|
-
ssl_context.set_params(
|
615
|
-
cert: OpenSSL::X509::Certificate.new(client_cert),
|
616
|
-
key: OpenSSL::PKey.read(client_cert_key)
|
617
|
-
)
|
618
|
-
elsif client_cert && !client_cert_key
|
619
|
-
raise ArgumentError, "Kafka client initialized with `ssl_client_cert` but no `ssl_client_cert_key`. Please provide both."
|
620
|
-
elsif !client_cert && client_cert_key
|
621
|
-
raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
|
622
|
-
end
|
623
|
-
|
624
|
-
if ca_cert || ca_cert_file_path || ssl_ca_certs_from_system
|
625
|
-
store = OpenSSL::X509::Store.new
|
626
|
-
Array(ca_cert).each do |cert|
|
627
|
-
store.add_cert(OpenSSL::X509::Certificate.new(cert))
|
628
|
-
end
|
629
|
-
if ca_cert_file_path
|
630
|
-
store.add_file(ca_cert_file_path)
|
631
|
-
end
|
632
|
-
if ssl_ca_certs_from_system
|
633
|
-
store.set_default_paths
|
634
|
-
end
|
635
|
-
ssl_context.cert_store = store
|
636
|
-
end
|
637
|
-
|
638
|
-
ssl_context
|
639
|
-
end
|
640
|
-
|
641
611
|
def normalize_seed_brokers(seed_brokers)
|
642
612
|
if seed_brokers.is_a?(String)
|
643
613
|
seed_brokers = seed_brokers.split(",")
|
644
614
|
end
|
645
615
|
|
646
|
-
seed_brokers.map
|
647
|
-
connection = "kafka://" + connection unless connection =~ /:\/\//
|
648
|
-
uri = URI.parse(connection)
|
649
|
-
uri.port ||= 9092 # Default Kafka port.
|
650
|
-
|
651
|
-
unless URI_SCHEMES.include?(uri.scheme)
|
652
|
-
raise Kafka::Error, "invalid protocol `#{uri.scheme}` in `#{connection}`"
|
653
|
-
end
|
654
|
-
|
655
|
-
uri
|
656
|
-
end
|
616
|
+
seed_brokers.map {|str| BrokerUri.parse(str) }
|
657
617
|
end
|
658
618
|
end
|
659
619
|
end
|
data/lib/kafka/cluster.rb
CHANGED
@@ -186,6 +186,11 @@ module Kafka
|
|
186
186
|
@logger.warn "Leader not yet available for `#{name}`, waiting 1s..."
|
187
187
|
sleep 1
|
188
188
|
|
189
|
+
retry
|
190
|
+
rescue Kafka::UnknownTopicOrPartition
|
191
|
+
@logger.warn "Topic `#{name}` not yet created, waiting 1s..."
|
192
|
+
sleep 1
|
193
|
+
|
189
194
|
retry
|
190
195
|
end
|
191
196
|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require "openssl"
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
module SslContext
|
5
|
+
|
6
|
+
def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, ca_certs_from_system: nil)
|
7
|
+
return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || ca_certs_from_system
|
8
|
+
|
9
|
+
ssl_context = OpenSSL::SSL::SSLContext.new
|
10
|
+
|
11
|
+
if client_cert && client_cert_key
|
12
|
+
ssl_context.set_params(
|
13
|
+
cert: OpenSSL::X509::Certificate.new(client_cert),
|
14
|
+
key: OpenSSL::PKey.read(client_cert_key)
|
15
|
+
)
|
16
|
+
elsif client_cert && !client_cert_key
|
17
|
+
raise ArgumentError, "Kafka client initialized with `ssl_client_cert` but no `ssl_client_cert_key`. Please provide both."
|
18
|
+
elsif !client_cert && client_cert_key
|
19
|
+
raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
|
20
|
+
end
|
21
|
+
|
22
|
+
if ca_cert || ca_cert_file_path || ca_certs_from_system
|
23
|
+
store = OpenSSL::X509::Store.new
|
24
|
+
Array(ca_cert).each do |cert|
|
25
|
+
store.add_cert(OpenSSL::X509::Certificate.new(cert))
|
26
|
+
end
|
27
|
+
if ca_cert_file_path
|
28
|
+
store.add_file(ca_cert_file_path)
|
29
|
+
end
|
30
|
+
if ca_certs_from_system
|
31
|
+
store.set_default_paths
|
32
|
+
end
|
33
|
+
ssl_context.cert_store = store
|
34
|
+
end
|
35
|
+
|
36
|
+
ssl_context
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-04-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -310,6 +310,7 @@ files:
|
|
310
310
|
- lib/kafka/async_producer.rb
|
311
311
|
- lib/kafka/broker.rb
|
312
312
|
- lib/kafka/broker_pool.rb
|
313
|
+
- lib/kafka/broker_uri.rb
|
313
314
|
- lib/kafka/client.rb
|
314
315
|
- lib/kafka/cluster.rb
|
315
316
|
- lib/kafka/compression.rb
|
@@ -382,6 +383,7 @@ files:
|
|
382
383
|
- lib/kafka/sasl_authenticator.rb
|
383
384
|
- lib/kafka/snappy_codec.rb
|
384
385
|
- lib/kafka/socket_with_timeout.rb
|
386
|
+
- lib/kafka/ssl_context.rb
|
385
387
|
- lib/kafka/ssl_socket_with_timeout.rb
|
386
388
|
- lib/kafka/statsd.rb
|
387
389
|
- lib/kafka/version.rb
|