ruby-kafka 0.7.6 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +102 -3
- data/.github/workflows/stale.yml +19 -0
- data/CHANGELOG.md +24 -0
- data/README.md +18 -0
- data/lib/kafka/async_producer.rb +3 -0
- data/lib/kafka/broker.rb +12 -0
- data/lib/kafka/client.rb +35 -3
- data/lib/kafka/cluster.rb +52 -0
- data/lib/kafka/compression.rb +13 -11
- data/lib/kafka/compressor.rb +1 -0
- data/lib/kafka/connection.rb +3 -0
- data/lib/kafka/consumer_group.rb +4 -1
- data/lib/kafka/datadog.rb +2 -10
- data/lib/kafka/fetched_batch.rb +5 -1
- data/lib/kafka/fetched_batch_generator.rb +4 -1
- data/lib/kafka/fetched_message.rb +1 -0
- data/lib/kafka/fetcher.rb +4 -1
- data/lib/kafka/gzip_codec.rb +4 -0
- data/lib/kafka/lz4_codec.rb +4 -0
- data/lib/kafka/producer.rb +20 -1
- data/lib/kafka/prometheus.rb +316 -0
- data/lib/kafka/protocol.rb +8 -0
- data/lib/kafka/protocol/add_offsets_to_txn_request.rb +29 -0
- data/lib/kafka/protocol/add_offsets_to_txn_response.rb +19 -0
- data/lib/kafka/protocol/join_group_request.rb +8 -2
- data/lib/kafka/protocol/offset_fetch_request.rb +3 -1
- data/lib/kafka/protocol/produce_request.rb +3 -1
- data/lib/kafka/protocol/record_batch.rb +5 -4
- data/lib/kafka/protocol/txn_offset_commit_request.rb +46 -0
- data/lib/kafka/protocol/txn_offset_commit_response.rb +18 -0
- data/lib/kafka/sasl/scram.rb +15 -12
- data/lib/kafka/snappy_codec.rb +4 -0
- data/lib/kafka/ssl_context.rb +4 -1
- data/lib/kafka/ssl_socket_with_timeout.rb +1 -0
- data/lib/kafka/tagged_logger.rb +25 -20
- data/lib/kafka/transaction_manager.rb +25 -0
- data/lib/kafka/version.rb +1 -1
- data/lib/kafka/zstd_codec.rb +27 -0
- data/ruby-kafka.gemspec +4 -2
- metadata +47 -6
data/lib/kafka/protocol.rb
CHANGED
@@ -33,7 +33,9 @@ module Kafka
|
|
33
33
|
DELETE_TOPICS_API = 20
|
34
34
|
INIT_PRODUCER_ID_API = 22
|
35
35
|
ADD_PARTITIONS_TO_TXN_API = 24
|
36
|
+
ADD_OFFSETS_TO_TXN_API = 25
|
36
37
|
END_TXN_API = 26
|
38
|
+
TXN_OFFSET_COMMIT_API = 28
|
37
39
|
DESCRIBE_CONFIGS_API = 32
|
38
40
|
ALTER_CONFIGS_API = 33
|
39
41
|
CREATE_PARTITIONS_API = 37
|
@@ -57,7 +59,9 @@ module Kafka
|
|
57
59
|
DELETE_TOPICS_API => :delete_topics,
|
58
60
|
INIT_PRODUCER_ID_API => :init_producer_id_api,
|
59
61
|
ADD_PARTITIONS_TO_TXN_API => :add_partitions_to_txn_api,
|
62
|
+
ADD_OFFSETS_TO_TXN_API => :add_offsets_to_txn_api,
|
60
63
|
END_TXN_API => :end_txn_api,
|
64
|
+
TXN_OFFSET_COMMIT_API => :txn_offset_commit_api,
|
61
65
|
DESCRIBE_CONFIGS_API => :describe_configs_api,
|
62
66
|
CREATE_PARTITIONS_API => :create_partitions
|
63
67
|
}
|
@@ -177,6 +181,10 @@ require "kafka/protocol/fetch_request"
|
|
177
181
|
require "kafka/protocol/fetch_response"
|
178
182
|
require "kafka/protocol/list_offset_request"
|
179
183
|
require "kafka/protocol/list_offset_response"
|
184
|
+
require "kafka/protocol/add_offsets_to_txn_request"
|
185
|
+
require "kafka/protocol/add_offsets_to_txn_response"
|
186
|
+
require "kafka/protocol/txn_offset_commit_request"
|
187
|
+
require "kafka/protocol/txn_offset_commit_response"
|
180
188
|
require "kafka/protocol/find_coordinator_request"
|
181
189
|
require "kafka/protocol/find_coordinator_response"
|
182
190
|
require "kafka/protocol/join_group_request"
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
module Protocol
|
5
|
+
class AddOffsetsToTxnRequest
|
6
|
+
def initialize(transactional_id: nil, producer_id:, producer_epoch:, group_id:)
|
7
|
+
@transactional_id = transactional_id
|
8
|
+
@producer_id = producer_id
|
9
|
+
@producer_epoch = producer_epoch
|
10
|
+
@group_id = group_id
|
11
|
+
end
|
12
|
+
|
13
|
+
def api_key
|
14
|
+
ADD_OFFSETS_TO_TXN_API
|
15
|
+
end
|
16
|
+
|
17
|
+
def response_class
|
18
|
+
AddOffsetsToTxnResponse
|
19
|
+
end
|
20
|
+
|
21
|
+
def encode(encoder)
|
22
|
+
encoder.write_string(@transactional_id.to_s)
|
23
|
+
encoder.write_int64(@producer_id)
|
24
|
+
encoder.write_int16(@producer_epoch)
|
25
|
+
encoder.write_string(@group_id)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Kafka
|
2
|
+
module Protocol
|
3
|
+
class AddOffsetsToTxnResponse
|
4
|
+
|
5
|
+
attr_reader :error_code
|
6
|
+
|
7
|
+
def initialize(error_code:)
|
8
|
+
@error_code = error_code
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.decode(decoder)
|
12
|
+
_throttle_time_ms = decoder.int32
|
13
|
+
error_code = decoder.int16
|
14
|
+
new(error_code: error_code)
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -7,13 +7,14 @@ module Kafka
|
|
7
7
|
class JoinGroupRequest
|
8
8
|
PROTOCOL_TYPE = "consumer"
|
9
9
|
|
10
|
-
def initialize(group_id:, session_timeout:, member_id:, topics: [])
|
10
|
+
def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [])
|
11
11
|
@group_id = group_id
|
12
12
|
@session_timeout = session_timeout * 1000 # Kafka wants ms.
|
13
|
+
@rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
|
13
14
|
@member_id = member_id || ""
|
14
15
|
@protocol_type = PROTOCOL_TYPE
|
15
16
|
@group_protocols = {
|
16
|
-
"
|
17
|
+
"roundrobin" => ConsumerGroupProtocol.new(topics: topics),
|
17
18
|
}
|
18
19
|
end
|
19
20
|
|
@@ -21,6 +22,10 @@ module Kafka
|
|
21
22
|
JOIN_GROUP_API
|
22
23
|
end
|
23
24
|
|
25
|
+
def api_version
|
26
|
+
1
|
27
|
+
end
|
28
|
+
|
24
29
|
def response_class
|
25
30
|
JoinGroupResponse
|
26
31
|
end
|
@@ -28,6 +33,7 @@ module Kafka
|
|
28
33
|
def encode(encoder)
|
29
34
|
encoder.write_string(@group_id)
|
30
35
|
encoder.write_int32(@session_timeout)
|
36
|
+
encoder.write_int32(@rebalance_timeout)
|
31
37
|
encoder.write_string(@member_id)
|
32
38
|
encoder.write_string(@protocol_type)
|
33
39
|
|
@@ -27,6 +27,8 @@ module Kafka
|
|
27
27
|
# Value => bytes
|
28
28
|
#
|
29
29
|
class ProduceRequest
|
30
|
+
API_MIN_VERSION = 3
|
31
|
+
|
30
32
|
attr_reader :transactional_id, :required_acks, :timeout, :messages_for_topics, :compressor
|
31
33
|
|
32
34
|
# @param required_acks [Integer]
|
@@ -45,7 +47,7 @@ module Kafka
|
|
45
47
|
end
|
46
48
|
|
47
49
|
def api_version
|
48
|
-
|
50
|
+
compressor.codec.nil? ? API_MIN_VERSION : [compressor.codec.produce_api_min_version, API_MIN_VERSION].max
|
49
51
|
end
|
50
52
|
|
51
53
|
def response_class
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'bigdecimal'
|
1
2
|
require 'digest/crc32'
|
2
3
|
require 'kafka/protocol/record'
|
3
4
|
|
@@ -131,7 +132,7 @@ module Kafka
|
|
131
132
|
|
132
133
|
records.each_with_index do |record, index|
|
133
134
|
record.offset_delta = index
|
134
|
-
record.timestamp_delta = (record.create_time - first_timestamp).to_i
|
135
|
+
record.timestamp_delta = ((record.create_time - first_timestamp) * 1000).to_i
|
135
136
|
end
|
136
137
|
@last_offset_delta = records.length - 1
|
137
138
|
end
|
@@ -167,8 +168,8 @@ module Kafka
|
|
167
168
|
log_append_time = (attributes & TIMESTAMP_TYPE_MASK) != 0
|
168
169
|
|
169
170
|
last_offset_delta = record_batch_decoder.int32
|
170
|
-
first_timestamp = Time.at(record_batch_decoder.int64 / 1000)
|
171
|
-
max_timestamp = Time.at(record_batch_decoder.int64 / 1000)
|
171
|
+
first_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
|
172
|
+
max_timestamp = Time.at(record_batch_decoder.int64 / BigDecimal(1000))
|
172
173
|
|
173
174
|
producer_id = record_batch_decoder.int64
|
174
175
|
producer_epoch = record_batch_decoder.int16
|
@@ -188,7 +189,7 @@ module Kafka
|
|
188
189
|
until records_array_decoder.eof?
|
189
190
|
record = Record.decode(records_array_decoder)
|
190
191
|
record.offset = first_offset + record.offset_delta
|
191
|
-
record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta
|
192
|
+
record.create_time = log_append_time && max_timestamp ? max_timestamp : first_timestamp + record.timestamp_delta / BigDecimal(1000)
|
192
193
|
records_array << record
|
193
194
|
end
|
194
195
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
module Protocol
|
5
|
+
class TxnOffsetCommitRequest
|
6
|
+
|
7
|
+
def api_key
|
8
|
+
TXN_OFFSET_COMMIT_API
|
9
|
+
end
|
10
|
+
|
11
|
+
def api_version
|
12
|
+
2
|
13
|
+
end
|
14
|
+
|
15
|
+
def response_class
|
16
|
+
TxnOffsetCommitResponse
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(transactional_id:, group_id:, producer_id:, producer_epoch:, offsets:)
|
20
|
+
@transactional_id = transactional_id
|
21
|
+
@producer_id = producer_id
|
22
|
+
@producer_epoch = producer_epoch
|
23
|
+
@group_id = group_id
|
24
|
+
@offsets = offsets
|
25
|
+
end
|
26
|
+
|
27
|
+
def encode(encoder)
|
28
|
+
encoder.write_string(@transactional_id.to_s)
|
29
|
+
encoder.write_string(@group_id)
|
30
|
+
encoder.write_int64(@producer_id)
|
31
|
+
encoder.write_int16(@producer_epoch)
|
32
|
+
|
33
|
+
encoder.write_array(@offsets) do |topic, partitions|
|
34
|
+
encoder.write_string(topic)
|
35
|
+
encoder.write_array(partitions) do |partition, offset|
|
36
|
+
encoder.write_int32(partition)
|
37
|
+
encoder.write_int64(offset[:offset])
|
38
|
+
encoder.write_string(nil) # metadata
|
39
|
+
encoder.write_int32(offset[:leader_epoch])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Kafka
|
2
|
+
module Protocol
|
3
|
+
class TxnOffsetCommitResponse
|
4
|
+
|
5
|
+
attr_reader :error_code
|
6
|
+
|
7
|
+
def initialize(error_code:)
|
8
|
+
@error_code = error_code
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.decode(decoder)
|
12
|
+
_throttle_time_ms = decoder.int32
|
13
|
+
error_code = decoder.int16
|
14
|
+
new(error_code: error_code)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/kafka/sasl/scram.rb
CHANGED
@@ -12,6 +12,7 @@ module Kafka
|
|
12
12
|
}.freeze
|
13
13
|
|
14
14
|
def initialize(username:, password:, mechanism: 'sha256', logger:)
|
15
|
+
@semaphore = Mutex.new
|
15
16
|
@username = username
|
16
17
|
@password = password
|
17
18
|
@logger = TaggedLogger.new(logger)
|
@@ -35,22 +36,24 @@ module Kafka
|
|
35
36
|
@logger.debug "Authenticating #{@username} with SASL #{@mechanism}"
|
36
37
|
|
37
38
|
begin
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
@semaphore.synchronize do
|
40
|
+
msg = first_message
|
41
|
+
@logger.debug "Sending first client SASL SCRAM message: #{msg}"
|
42
|
+
encoder.write_bytes(msg)
|
41
43
|
|
42
|
-
|
43
|
-
|
44
|
+
@server_first_message = decoder.bytes
|
45
|
+
@logger.debug "Received first server SASL SCRAM message: #{@server_first_message}"
|
44
46
|
|
45
|
-
|
46
|
-
|
47
|
-
|
47
|
+
msg = final_message
|
48
|
+
@logger.debug "Sending final client SASL SCRAM message: #{msg}"
|
49
|
+
encoder.write_bytes(msg)
|
48
50
|
|
49
|
-
|
50
|
-
|
51
|
+
response = parse_response(decoder.bytes)
|
52
|
+
@logger.debug "Received last server SASL SCRAM message: #{response}"
|
51
53
|
|
52
|
-
|
53
|
-
|
54
|
+
raise FailedScramAuthentication, response['e'] if response['e']
|
55
|
+
raise FailedScramAuthentication, "Invalid server signature" if response['v'] != server_signature
|
56
|
+
end
|
54
57
|
rescue EOFError => e
|
55
58
|
raise FailedScramAuthentication, e.message
|
56
59
|
end
|
data/lib/kafka/snappy_codec.rb
CHANGED
data/lib/kafka/ssl_context.rb
CHANGED
@@ -6,7 +6,7 @@ module Kafka
|
|
6
6
|
module SslContext
|
7
7
|
CLIENT_CERT_DELIMITER = "\n-----END CERTIFICATE-----\n"
|
8
8
|
|
9
|
-
def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_key_password: nil, client_cert_chain: nil, ca_certs_from_system: nil)
|
9
|
+
def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_key_password: nil, client_cert_chain: nil, ca_certs_from_system: nil, verify_hostname: true)
|
10
10
|
return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || client_cert_key_password || client_cert_chain || ca_certs_from_system
|
11
11
|
|
12
12
|
ssl_context = OpenSSL::SSL::SSLContext.new
|
@@ -54,6 +54,9 @@ module Kafka
|
|
54
54
|
store.set_default_paths
|
55
55
|
end
|
56
56
|
ssl_context.cert_store = store
|
57
|
+
ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
58
|
+
# Verify certificate hostname if supported (ruby >= 2.4.0)
|
59
|
+
ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
|
57
60
|
end
|
58
61
|
|
59
62
|
ssl_context
|
@@ -57,6 +57,7 @@ module Kafka
|
|
57
57
|
|
58
58
|
# once that's connected, we can start initiating the ssl socket
|
59
59
|
@ssl_socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, ssl_context)
|
60
|
+
@ssl_socket.hostname = host
|
60
61
|
|
61
62
|
begin
|
62
63
|
# Initiate the socket connection in the background. If it doesn't fail
|
data/lib/kafka/tagged_logger.rb
CHANGED
@@ -1,13 +1,20 @@
|
|
1
|
-
require 'forwardable'
|
2
|
-
|
3
1
|
# Basic implementation of a tagged logger that matches the API of
|
4
2
|
# ActiveSupport::TaggedLogging.
|
5
3
|
|
4
|
+
require 'delegate'
|
5
|
+
require 'logger'
|
6
|
+
|
6
7
|
module Kafka
|
7
|
-
|
8
|
+
class TaggedLogger < SimpleDelegator
|
8
9
|
|
9
|
-
|
10
|
-
|
10
|
+
%i(debug info warn error).each do |method|
|
11
|
+
define_method method do |msg_or_progname, &block|
|
12
|
+
if block_given?
|
13
|
+
super(msg_or_progname, &block)
|
14
|
+
else
|
15
|
+
super("#{tags_text}#{msg_or_progname}")
|
16
|
+
end
|
17
|
+
end
|
11
18
|
end
|
12
19
|
|
13
20
|
def tagged(*tags)
|
@@ -44,23 +51,21 @@ module Kafka
|
|
44
51
|
end
|
45
52
|
end
|
46
53
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
delegate [:push_tags, :pop_tags, :clear_tags!] => :formatter
|
52
|
-
|
53
|
-
def self.new(logger)
|
54
|
-
logger ||= Logger.new(nil)
|
55
|
-
return logger if logger.respond_to?(:push_tags) # already included
|
56
|
-
# Ensure we set a default formatter so we aren't extending nil!
|
57
|
-
logger.formatter ||= Logger::Formatter.new
|
58
|
-
logger.formatter.extend TaggedFormatter
|
59
|
-
logger.extend(self)
|
54
|
+
def self.new(logger_or_stream = nil)
|
55
|
+
# don't keep wrapping the same logger over and over again
|
56
|
+
return logger_or_stream if logger_or_stream.is_a?(TaggedLogger)
|
57
|
+
super
|
60
58
|
end
|
61
59
|
|
62
|
-
def
|
63
|
-
|
60
|
+
def initialize(logger_or_stream = nil)
|
61
|
+
logger = if %w(info debug warn error).all? { |s| logger_or_stream.respond_to?(s) }
|
62
|
+
logger_or_stream
|
63
|
+
elsif logger_or_stream
|
64
|
+
::Logger.new(logger_or_stream)
|
65
|
+
else
|
66
|
+
::Logger.new(nil)
|
67
|
+
end
|
68
|
+
super(logger)
|
64
69
|
end
|
65
70
|
|
66
71
|
def flush
|
@@ -217,6 +217,31 @@ module Kafka
|
|
217
217
|
raise
|
218
218
|
end
|
219
219
|
|
220
|
+
def send_offsets_to_txn(offsets:, group_id:)
|
221
|
+
force_transactional!
|
222
|
+
|
223
|
+
unless @transaction_state.in_transaction?
|
224
|
+
raise 'Transaction is not valid to send offsets'
|
225
|
+
end
|
226
|
+
|
227
|
+
add_response = transaction_coordinator.add_offsets_to_txn(
|
228
|
+
transactional_id: @transactional_id,
|
229
|
+
producer_id: @producer_id,
|
230
|
+
producer_epoch: @producer_epoch,
|
231
|
+
group_id: group_id
|
232
|
+
)
|
233
|
+
Protocol.handle_error(add_response.error_code)
|
234
|
+
|
235
|
+
send_response = transaction_coordinator.txn_offset_commit(
|
236
|
+
transactional_id: @transactional_id,
|
237
|
+
group_id: group_id,
|
238
|
+
producer_id: @producer_id,
|
239
|
+
producer_epoch: @producer_epoch,
|
240
|
+
offsets: offsets
|
241
|
+
)
|
242
|
+
Protocol.handle_error(send_response.error_code)
|
243
|
+
end
|
244
|
+
|
220
245
|
def in_transaction?
|
221
246
|
@transaction_state.in_transaction?
|
222
247
|
end
|
data/lib/kafka/version.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
class ZstdCodec
|
5
|
+
def codec_id
|
6
|
+
4
|
7
|
+
end
|
8
|
+
|
9
|
+
def produce_api_min_version
|
10
|
+
7
|
11
|
+
end
|
12
|
+
|
13
|
+
def load
|
14
|
+
require "zstd-ruby"
|
15
|
+
rescue LoadError
|
16
|
+
raise LoadError, "using zstd compression requires adding a dependency on the `zstd-ruby` gem to your Gemfile."
|
17
|
+
end
|
18
|
+
|
19
|
+
def compress(data)
|
20
|
+
Zstd.compress(data)
|
21
|
+
end
|
22
|
+
|
23
|
+
def decompress(data)
|
24
|
+
Zstd.decompress(data)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|