ruby-kafka 0.5.0 → 0.5.1.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +33 -0
- data/CHANGELOG.md +8 -0
- data/README.md +29 -7
- data/docker-compose.yml +39 -0
- data/lib/kafka.rb +14 -0
- data/lib/kafka/broker.rb +48 -19
- data/lib/kafka/broker_pool.rb +3 -1
- data/lib/kafka/client.rb +65 -7
- data/lib/kafka/cluster.rb +68 -4
- data/lib/kafka/connection.rb +8 -18
- data/lib/kafka/connection_builder.rb +2 -1
- data/lib/kafka/consumer.rb +16 -4
- data/lib/kafka/fetch_operation.rb +3 -1
- data/lib/kafka/protocol.rb +8 -0
- data/lib/kafka/protocol/api_versions_request.rb +19 -0
- data/lib/kafka/protocol/api_versions_response.rb +47 -0
- data/lib/kafka/protocol/create_topics_request.rb +40 -0
- data/lib/kafka/protocol/create_topics_response.rb +24 -0
- data/lib/kafka/protocol/decoder.rb +9 -0
- data/lib/kafka/protocol/encoder.rb +8 -0
- data/lib/kafka/protocol/fetch_request.rb +4 -2
- data/lib/kafka/protocol/message.rb +13 -1
- data/lib/kafka/protocol/message_set.rb +7 -1
- data/lib/kafka/protocol/metadata_response.rb +10 -2
- data/lib/kafka/protocol/sasl_handshake_request.rb +1 -1
- data/lib/kafka/protocol/topic_metadata_request.rb +4 -0
- data/lib/kafka/round_robin_assignment_strategy.rb +5 -1
- data/lib/kafka/sasl/gssapi.rb +74 -0
- data/lib/kafka/sasl/plain.rb +37 -0
- data/lib/kafka/sasl/scram.rb +175 -0
- data/lib/kafka/sasl_authenticator.rb +31 -39
- data/lib/kafka/version.rb +1 -1
- metadata +14 -8
- data/circle.yml +0 -23
- data/lib/kafka/sasl_gssapi_authenticator.rb +0 -77
- data/lib/kafka/sasl_plain_authenticator.rb +0 -37
@@ -1,56 +1,48 @@
|
|
1
|
-
require 'kafka/
|
2
|
-
require 'kafka/
|
1
|
+
require 'kafka/sasl/plain'
|
2
|
+
require 'kafka/sasl/gssapi'
|
3
|
+
require 'kafka/sasl/scram'
|
3
4
|
|
4
5
|
module Kafka
|
5
6
|
class SaslAuthenticator
|
6
|
-
def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
|
7
|
+
def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
|
8
|
+
sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:,
|
9
|
+
sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:)
|
7
10
|
@logger = logger
|
8
|
-
@sasl_gssapi_principal = sasl_gssapi_principal
|
9
|
-
@sasl_gssapi_keytab = sasl_gssapi_keytab
|
10
|
-
@sasl_plain_authzid = sasl_plain_authzid
|
11
|
-
@sasl_plain_username = sasl_plain_username
|
12
|
-
@sasl_plain_password = sasl_plain_password
|
13
|
-
end
|
14
|
-
|
15
|
-
def authenticate!(connection)
|
16
|
-
if authenticate_using_sasl_gssapi?
|
17
|
-
sasl_gssapi_authenticate(connection)
|
18
|
-
elsif authenticate_using_sasl_plain?
|
19
|
-
sasl_plain_authenticate(connection)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
private
|
24
11
|
|
25
|
-
|
26
|
-
|
27
|
-
|
12
|
+
@plain = Sasl::Plain.new(
|
13
|
+
authzid: sasl_plain_authzid,
|
14
|
+
username: sasl_plain_username,
|
15
|
+
password: sasl_plain_password,
|
28
16
|
logger: @logger,
|
29
|
-
sasl_gssapi_principal: @sasl_gssapi_principal,
|
30
|
-
sasl_gssapi_keytab: @sasl_gssapi_keytab
|
31
17
|
)
|
32
18
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
def sasl_plain_authenticate(connection)
|
37
|
-
auth = SaslPlainAuthenticator.new(
|
38
|
-
connection: connection,
|
19
|
+
@gssapi = Sasl::Gssapi.new(
|
20
|
+
principal: sasl_gssapi_principal,
|
21
|
+
keytab: sasl_gssapi_keytab,
|
39
22
|
logger: @logger,
|
40
|
-
authzid: @sasl_plain_authzid,
|
41
|
-
username: @sasl_plain_username,
|
42
|
-
password: @sasl_plain_password
|
43
23
|
)
|
44
24
|
|
45
|
-
|
25
|
+
@scram = Sasl::Scram.new(
|
26
|
+
username: sasl_scram_username,
|
27
|
+
password: sasl_scram_password,
|
28
|
+
mechanism: sasl_scram_mechanism,
|
29
|
+
logger: @logger,
|
30
|
+
)
|
46
31
|
end
|
47
32
|
|
48
|
-
def
|
49
|
-
|
50
|
-
|
33
|
+
def authenticate!(connection)
|
34
|
+
mechanism = [@gssapi, @plain, @scram].find(&:configured?)
|
35
|
+
|
36
|
+
return if mechanism.nil?
|
37
|
+
|
38
|
+
ident = mechanism.ident
|
39
|
+
response = connection.send_request(Kafka::Protocol::SaslHandshakeRequest.new(ident))
|
40
|
+
|
41
|
+
unless response.error_code == 0 && response.enabled_mechanisms.include?(ident)
|
42
|
+
raise Kafka::Error, "#{ident} is not supported."
|
43
|
+
end
|
51
44
|
|
52
|
-
|
53
|
-
@sasl_plain_authzid && @sasl_plain_username && @sasl_plain_password
|
45
|
+
mechanism.authenticate!(connection.to_s, connection.encoder, connection.decoder)
|
54
46
|
end
|
55
47
|
end
|
56
48
|
end
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.1.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -255,6 +255,7 @@ executables: []
|
|
255
255
|
extensions: []
|
256
256
|
extra_rdoc_files: []
|
257
257
|
files:
|
258
|
+
- ".circleci/config.yml"
|
258
259
|
- ".gitignore"
|
259
260
|
- ".rspec"
|
260
261
|
- ".rubocop.yml"
|
@@ -271,7 +272,7 @@ files:
|
|
271
272
|
- ci/consumer.rb
|
272
273
|
- ci/init.rb
|
273
274
|
- ci/producer.rb
|
274
|
-
-
|
275
|
+
- docker-compose.yml
|
275
276
|
- examples/consumer-group.rb
|
276
277
|
- examples/firehose-consumer.rb
|
277
278
|
- examples/firehose-producer.rb
|
@@ -305,7 +306,11 @@ files:
|
|
305
306
|
- lib/kafka/produce_operation.rb
|
306
307
|
- lib/kafka/producer.rb
|
307
308
|
- lib/kafka/protocol.rb
|
309
|
+
- lib/kafka/protocol/api_versions_request.rb
|
310
|
+
- lib/kafka/protocol/api_versions_response.rb
|
308
311
|
- lib/kafka/protocol/consumer_group_protocol.rb
|
312
|
+
- lib/kafka/protocol/create_topics_request.rb
|
313
|
+
- lib/kafka/protocol/create_topics_response.rb
|
309
314
|
- lib/kafka/protocol/decoder.rb
|
310
315
|
- lib/kafka/protocol/encoder.rb
|
311
316
|
- lib/kafka/protocol/fetch_request.rb
|
@@ -337,9 +342,10 @@ files:
|
|
337
342
|
- lib/kafka/protocol/sync_group_response.rb
|
338
343
|
- lib/kafka/protocol/topic_metadata_request.rb
|
339
344
|
- lib/kafka/round_robin_assignment_strategy.rb
|
345
|
+
- lib/kafka/sasl/gssapi.rb
|
346
|
+
- lib/kafka/sasl/plain.rb
|
347
|
+
- lib/kafka/sasl/scram.rb
|
340
348
|
- lib/kafka/sasl_authenticator.rb
|
341
|
-
- lib/kafka/sasl_gssapi_authenticator.rb
|
342
|
-
- lib/kafka/sasl_plain_authenticator.rb
|
343
349
|
- lib/kafka/snappy_codec.rb
|
344
350
|
- lib/kafka/socket_with_timeout.rb
|
345
351
|
- lib/kafka/ssl_socket_with_timeout.rb
|
@@ -376,12 +382,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
376
382
|
version: 2.1.0
|
377
383
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
378
384
|
requirements:
|
379
|
-
- - "
|
385
|
+
- - ">"
|
380
386
|
- !ruby/object:Gem::Version
|
381
|
-
version:
|
387
|
+
version: 1.3.1
|
382
388
|
requirements: []
|
383
389
|
rubyforge_project:
|
384
|
-
rubygems_version: 2.6.
|
390
|
+
rubygems_version: 2.6.13
|
385
391
|
signing_key:
|
386
392
|
specification_version: 4
|
387
393
|
summary: A client library for the Kafka distributed commit log.
|
data/circle.yml
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
machine:
|
2
|
-
pre:
|
3
|
-
- curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
|
4
|
-
services:
|
5
|
-
- docker
|
6
|
-
environment:
|
7
|
-
LOG_LEVEL: DEBUG
|
8
|
-
ruby:
|
9
|
-
version: 2.4.1
|
10
|
-
|
11
|
-
dependencies:
|
12
|
-
pre:
|
13
|
-
- docker -v
|
14
|
-
- docker pull ches/kafka:0.10.0.0
|
15
|
-
- docker pull jplock/zookeeper:3.4.6
|
16
|
-
|
17
|
-
test:
|
18
|
-
override:
|
19
|
-
- bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
|
20
|
-
- bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/functional.xml --tag functional
|
21
|
-
post:
|
22
|
-
- bundle exec rubocop
|
23
|
-
- cp *.log $CIRCLE_ARTIFACTS/ || true
|
@@ -1,77 +0,0 @@
|
|
1
|
-
module Kafka
|
2
|
-
class SaslGssapiAuthenticator
|
3
|
-
GSSAPI_IDENT = "GSSAPI"
|
4
|
-
GSSAPI_CONFIDENTIALITY = false
|
5
|
-
|
6
|
-
def initialize(connection:, logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:)
|
7
|
-
@connection = connection
|
8
|
-
@logger = logger
|
9
|
-
@principal = sasl_gssapi_principal
|
10
|
-
@keytab = sasl_gssapi_keytab
|
11
|
-
|
12
|
-
load_gssapi
|
13
|
-
initialize_gssapi_context
|
14
|
-
end
|
15
|
-
|
16
|
-
def authenticate!
|
17
|
-
proceed_sasl_gssapi_negotiation
|
18
|
-
end
|
19
|
-
|
20
|
-
private
|
21
|
-
|
22
|
-
def proceed_sasl_gssapi_negotiation
|
23
|
-
response = @connection.send_request(Kafka::Protocol::SaslHandshakeRequest.new(GSSAPI_IDENT))
|
24
|
-
|
25
|
-
@encoder = @connection.encoder
|
26
|
-
@decoder = @connection.decoder
|
27
|
-
|
28
|
-
unless response.error_code == 0 && response.enabled_mechanisms.include?(GSSAPI_IDENT)
|
29
|
-
raise Kafka::Error, "#{GSSAPI_IDENT} is not supported."
|
30
|
-
end
|
31
|
-
|
32
|
-
# send gssapi token and receive token to verify
|
33
|
-
token_to_verify = send_and_receive_sasl_token
|
34
|
-
|
35
|
-
# verify incoming token
|
36
|
-
unless @gssapi_ctx.init_context(token_to_verify)
|
37
|
-
raise Kafka::Error, "GSSAPI context verification failed."
|
38
|
-
end
|
39
|
-
|
40
|
-
# we can continue, so send OK
|
41
|
-
@encoder.write([0, 2].pack('l>c'))
|
42
|
-
|
43
|
-
# read wrapped message and return it back with principal
|
44
|
-
handshake_messages
|
45
|
-
end
|
46
|
-
|
47
|
-
def handshake_messages
|
48
|
-
msg = @decoder.bytes
|
49
|
-
raise Kafka::Error, "GSSAPI negotiation failed." unless msg
|
50
|
-
# unwrap with integrity only
|
51
|
-
msg_unwrapped = @gssapi_ctx.unwrap_message(msg, GSSAPI_CONFIDENTIALITY)
|
52
|
-
msg_wrapped = @gssapi_ctx.wrap_message(msg_unwrapped + @principal, GSSAPI_CONFIDENTIALITY)
|
53
|
-
@encoder.write_bytes(msg_wrapped)
|
54
|
-
end
|
55
|
-
|
56
|
-
def send_and_receive_sasl_token
|
57
|
-
@encoder.write_bytes(@gssapi_token)
|
58
|
-
@decoder.bytes
|
59
|
-
end
|
60
|
-
|
61
|
-
def load_gssapi
|
62
|
-
begin
|
63
|
-
require "gssapi"
|
64
|
-
rescue LoadError
|
65
|
-
@logger.error "In order to use GSSAPI authentication you need to install the `gssapi` gem."
|
66
|
-
raise
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def initialize_gssapi_context
|
71
|
-
@logger.debug "GSSAPI: Initializing context with #{@connection.to_s}, principal #{@principal}"
|
72
|
-
|
73
|
-
@gssapi_ctx = GSSAPI::Simple.new(@connection.to_s, @principal, @keytab)
|
74
|
-
@gssapi_token = @gssapi_ctx.init_context(nil)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
module Kafka
|
2
|
-
class SaslPlainAuthenticator
|
3
|
-
PLAIN_IDENT = "PLAIN"
|
4
|
-
|
5
|
-
def initialize(connection:, logger:, authzid:, username:, password:)
|
6
|
-
@connection = connection
|
7
|
-
@logger = logger
|
8
|
-
@authzid = authzid
|
9
|
-
@username = username
|
10
|
-
@password = password
|
11
|
-
end
|
12
|
-
|
13
|
-
def authenticate!
|
14
|
-
response = @connection.send_request(Kafka::Protocol::SaslHandshakeRequest.new(PLAIN_IDENT))
|
15
|
-
|
16
|
-
@encoder = @connection.encoder
|
17
|
-
@decoder = @connection.decoder
|
18
|
-
|
19
|
-
unless response.error_code == 0 && response.enabled_mechanisms.include?(PLAIN_IDENT)
|
20
|
-
raise Kafka::Error, "#{PLAIN_IDENT} is not supported."
|
21
|
-
end
|
22
|
-
|
23
|
-
# SASL PLAIN
|
24
|
-
msg = [@authzid.to_s,
|
25
|
-
@username.to_s,
|
26
|
-
@password.to_s].join("\000").force_encoding('utf-8')
|
27
|
-
@encoder.write_bytes(msg)
|
28
|
-
begin
|
29
|
-
msg = @decoder.bytes
|
30
|
-
raise Kafka::Error, 'SASL PLAIN authentication failed: unknown error' unless msg
|
31
|
-
rescue Errno::ETIMEDOUT, EOFError => e
|
32
|
-
raise Kafka::Error, "SASL PLAIN authentication failed: #{e.message}"
|
33
|
-
end
|
34
|
-
@logger.debug 'SASL PLAIN authentication successful.'
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|