ruby-kafka 0.7.6 → 0.7.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '08731d84b1c82d5bf46625993d552e80face5fa5dbf71af68ff60cb454f740bf'
4
- data.tar.gz: 64aaa4a1b1dcd4200e058bfa1157d07749044752362ff296b5eadd6108c365da
3
+ metadata.gz: fe93873554ad9e17df699fa49612e0dae938a498c086ae765bf83b79f4a8106f
4
+ data.tar.gz: eb56576533a3b5d1983e755fc0399464315a47f43d6fc68d9ef20c4df5f6d351
5
5
  SHA512:
6
- metadata.gz: d154e9ecbbea7014d33ef93b6e21b1897df7628abb51a04094c2209518b43097414b54f03175fa0d59f89368ca2821559b9739c054503046ea74ef6cd875a4d6
7
- data.tar.gz: 23680e5dadc11ae5a810708a8d2d7de8e5dbeb0f964164b2d29288737bcd5fa4c9885d10b9660a538a6b9faa27d1accf83af44dabce4ef6d2c56f6b105bb9c91
6
+ metadata.gz: 19158030f7e028cb5dc350a405867273dadfaaa81208f4684c8c6a68b4d4b2507f97ec13b171e58da1b0277fa6712ccb3db50735cd0db5dd3d9839c0ddcba18b
7
+ data.tar.gz: d37f2e3ceb765ce5bf267ddcdbcf524251de9356b4e0fa1d16562cf762b99047146c2cda80482792b3f4845b789a8cf8446613f9e49990a22cf73844b9380714
@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ ## 0.7.7
8
+ - Producer send offsets in transaction (#723)
9
+ - Support zstd compression (#724)
10
+ - Verify SSL Certificates (#730)
11
+
7
12
  ## 0.7.6
8
13
  - Introduce regex matching in `Consumer#subscribe` (#700)
9
14
  - Only rejoin group on error if we're not in shutdown mode (#711)
data/README.md CHANGED
@@ -424,6 +424,7 @@ Compression is enabled by passing the `compression_codec` parameter to `#produce
424
424
  * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
425
425
  * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
426
426
  * `:lz4` for [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) compression.
427
+ * `:zstd` for [zstd](https://facebook.github.io/zstd/) compression.
427
428
 
428
429
  By default, all message sets will be compressed if you specify a compression codec. To increase the compression threshold, set `compression_threshold` to an integer value higher than one.
429
430
 
@@ -182,6 +182,18 @@ module Kafka
182
182
  send_request(request)
183
183
  end
184
184
 
185
+ def add_offsets_to_txn(**options)
186
+ request = Protocol::AddOffsetsToTxnRequest.new(**options)
187
+
188
+ send_request(request)
189
+ end
190
+
191
+ def txn_offset_commit(**options)
192
+ request = Protocol::TxnOffsetCommitRequest.new(**options)
193
+
194
+ send_request(request)
195
+ end
196
+
185
197
  private
186
198
 
187
199
  def send_request(request)
@@ -233,8 +233,8 @@ module Kafka
233
233
  # result in {BufferOverflow} being raised.
234
234
  #
235
235
  # @param compression_codec [Symbol, nil] the name of the compression codec to
236
- # use, or nil if no compression should be performed. Valid codecs: `:snappy`
237
- # and `:gzip`.
236
+ # use, or nil if no compression should be performed. Valid codecs: `:snappy`,
237
+ # `:gzip`, `:lz4`, `:zstd`
238
238
  #
239
239
  # @param compression_threshold [Integer] the number of messages that needs to
240
240
  # be in a message set before it should be compressed. Note that message sets
@@ -3,27 +3,27 @@
3
3
  require "kafka/snappy_codec"
4
4
  require "kafka/gzip_codec"
5
5
  require "kafka/lz4_codec"
6
+ require "kafka/zstd_codec"
6
7
 
7
8
  module Kafka
8
9
  module Compression
9
- CODEC_NAMES = {
10
- 1 => :gzip,
11
- 2 => :snappy,
12
- 3 => :lz4,
13
- }.freeze
14
-
15
- CODECS = {
10
+ CODECS_BY_NAME = {
16
11
  :gzip => GzipCodec.new,
17
12
  :snappy => SnappyCodec.new,
18
13
  :lz4 => LZ4Codec.new,
14
+ :zstd => ZstdCodec.new,
19
15
  }.freeze
20
16
 
17
+ CODECS_BY_ID = CODECS_BY_NAME.each_with_object({}) do |(_, codec), hash|
18
+ hash[codec.codec_id] = codec
19
+ end.freeze
20
+
21
21
  def self.codecs
22
- CODECS.keys
22
+ CODECS_BY_NAME.keys
23
23
  end
24
24
 
25
25
  def self.find_codec(name)
26
- codec = CODECS.fetch(name) do
26
+ codec = CODECS_BY_NAME.fetch(name) do
27
27
  raise "Unknown compression codec #{name}"
28
28
  end
29
29
 
@@ -33,11 +33,13 @@ module Kafka
33
33
  end
34
34
 
35
35
  def self.find_codec_by_id(codec_id)
36
- codec_name = CODEC_NAMES.fetch(codec_id) do
36
+ codec = CODECS_BY_ID.fetch(codec_id) do
37
37
  raise "Unknown codec id #{codec_id}"
38
38
  end
39
39
 
40
- find_codec(codec_name)
40
+ codec.load
41
+
42
+ codec
41
43
  end
42
44
  end
43
45
  end
@@ -18,6 +18,7 @@ module Kafka
18
18
  # * `compressed_bytesize` – the byte size of the compressed data.
19
19
  #
20
20
  class Compressor
21
+ attr_reader :codec
21
22
 
22
23
  # @param codec_name [Symbol, nil]
23
24
  # @param threshold [Integer] the minimum number of messages in a message set
@@ -13,18 +13,22 @@ module Kafka
13
13
  # @return [Integer]
14
14
  attr_reader :last_offset
15
15
 
16
+ # @return [Integer]
17
+ attr_reader :leader_epoch
18
+
16
19
  # @return [Integer] the offset of the most recent message in the partition.
17
20
  attr_reader :highwater_mark_offset
18
21
 
19
22
  # @return [Array<Kafka::FetchedMessage>]
20
23
  attr_accessor :messages
21
24
 
22
- def initialize(topic:, partition:, highwater_mark_offset:, messages:, last_offset: nil)
25
+ def initialize(topic:, partition:, highwater_mark_offset:, messages:, last_offset: nil, leader_epoch: nil)
23
26
  @topic = topic
24
27
  @partition = partition
25
28
  @highwater_mark_offset = highwater_mark_offset
26
29
  @messages = messages
27
30
  @last_offset = last_offset
31
+ @leader_epoch = leader_epoch
28
32
  end
29
33
 
30
34
  def empty?
@@ -62,11 +62,13 @@ module Kafka
62
62
  def extract_records
63
63
  records = []
64
64
  last_offset = nil
65
+ leader_epoch = nil
65
66
  aborted_transactions = @fetched_partition.aborted_transactions.sort_by(&:first_offset)
66
67
  aborted_producer_ids = {}
67
68
 
68
69
  @fetched_partition.messages.each do |record_batch|
69
70
  last_offset = record_batch.last_offset if last_offset.nil? || last_offset < record_batch.last_offset
71
+ leader_epoch = record_batch.partition_leader_epoch if leader_epoch.nil? || leader_epoch < record_batch.partition_leader_epoch
70
72
  # Find the list of aborted producer IDs less than current offset
71
73
  unless aborted_transactions.empty?
72
74
  if aborted_transactions.first.first_offset <= record_batch.last_offset
@@ -99,6 +101,7 @@ module Kafka
99
101
  topic: @topic,
100
102
  partition: @fetched_partition.partition,
101
103
  last_offset: last_offset,
104
+ leader_epoch: leader_epoch,
102
105
  highwater_mark_offset: @fetched_partition.highwater_mark_offset,
103
106
  messages: records
104
107
  )
@@ -43,5 +43,6 @@ module Kafka
43
43
  def is_control_record
44
44
  @message.is_control_record
45
45
  end
46
+
46
47
  end
47
48
  end
@@ -6,6 +6,10 @@ module Kafka
6
6
  1
7
7
  end
8
8
 
9
+ def produce_api_min_version
10
+ 0
11
+ end
12
+
9
13
  def load
10
14
  require "zlib"
11
15
  end
@@ -6,6 +6,10 @@ module Kafka
6
6
  3
7
7
  end
8
8
 
9
+ def produce_api_min_version
10
+ 0
11
+ end
12
+
9
13
  def load
10
14
  require "extlz4"
11
15
  rescue LoadError
@@ -68,6 +68,8 @@ module Kafka
68
68
  #
69
69
  # * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
70
70
  # * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
71
+ # * `:lz4` for [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) compression.
72
+ # * `:zstd` for [zstd](https://facebook.github.io/zstd/) compression.
71
73
  #
72
74
  # By default, all message sets will be compressed if you specify a compression
73
75
  # codec. To increase the compression threshold, set `compression_threshold` to
@@ -328,6 +330,20 @@ module Kafka
328
330
  @transaction_manager.abort_transaction
329
331
  end
330
332
 
333
+ # Sends batch last offset to the consumer group coordinator, and also marks
334
+ # this offset as part of the current transaction. This offset will be considered
335
+ # committed only if the transaction is committed successfully.
336
+ #
337
+ # This method should be used when you need to batch consumed and produced messages
338
+ # together, typically in a consume-transform-produce pattern. Thus, the specified
339
+ # group_id should be the same as config parameter group_id of the used
340
+ # consumer.
341
+ #
342
+ # @return [nil]
343
+ def send_offsets_to_transaction(batch:, group_id:)
344
+ @transaction_manager.send_offsets_to_txn(offsets: { batch.topic => { batch.partition => { offset: batch.last_offset + 1, leader_epoch: batch.leader_epoch } } }, group_id: group_id)
345
+ end
346
+
331
347
  # Syntactic sugar to enable easier transaction usage. Do the following steps
332
348
  #
333
349
  # - Start the transaction (with Producer#begin_transaction)
@@ -33,7 +33,9 @@ module Kafka
33
33
  DELETE_TOPICS_API = 20
34
34
  INIT_PRODUCER_ID_API = 22
35
35
  ADD_PARTITIONS_TO_TXN_API = 24
36
+ ADD_OFFSETS_TO_TXN_API = 25
36
37
  END_TXN_API = 26
38
+ TXN_OFFSET_COMMIT_API = 28
37
39
  DESCRIBE_CONFIGS_API = 32
38
40
  ALTER_CONFIGS_API = 33
39
41
  CREATE_PARTITIONS_API = 37
@@ -57,7 +59,9 @@ module Kafka
57
59
  DELETE_TOPICS_API => :delete_topics,
58
60
  INIT_PRODUCER_ID_API => :init_producer_id_api,
59
61
  ADD_PARTITIONS_TO_TXN_API => :add_partitions_to_txn_api,
62
+ ADD_OFFSETS_TO_TXN_API => :add_offsets_to_txn_api,
60
63
  END_TXN_API => :end_txn_api,
64
+ TXN_OFFSET_COMMIT_API => :txn_offset_commit_api,
61
65
  DESCRIBE_CONFIGS_API => :describe_configs_api,
62
66
  CREATE_PARTITIONS_API => :create_partitions
63
67
  }
@@ -177,6 +181,10 @@ require "kafka/protocol/fetch_request"
177
181
  require "kafka/protocol/fetch_response"
178
182
  require "kafka/protocol/list_offset_request"
179
183
  require "kafka/protocol/list_offset_response"
184
+ require "kafka/protocol/add_offsets_to_txn_request"
185
+ require "kafka/protocol/add_offsets_to_txn_response"
186
+ require "kafka/protocol/txn_offset_commit_request"
187
+ require "kafka/protocol/txn_offset_commit_response"
180
188
  require "kafka/protocol/find_coordinator_request"
181
189
  require "kafka/protocol/find_coordinator_response"
182
190
  require "kafka/protocol/join_group_request"
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kafka
4
+ module Protocol
5
+ class AddOffsetsToTxnRequest
6
+ def initialize(transactional_id: nil, producer_id:, producer_epoch:, group_id:)
7
+ @transactional_id = transactional_id
8
+ @producer_id = producer_id
9
+ @producer_epoch = producer_epoch
10
+ @group_id = group_id
11
+ end
12
+
13
+ def api_key
14
+ ADD_OFFSETS_TO_TXN_API
15
+ end
16
+
17
+ def response_class
18
+ AddOffsetsToTxnResponse
19
+ end
20
+
21
+ def encode(encoder)
22
+ encoder.write_string(@transactional_id.to_s)
23
+ encoder.write_int64(@producer_id)
24
+ encoder.write_int16(@producer_epoch)
25
+ encoder.write_string(@group_id)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,19 @@
1
+ module Kafka
2
+ module Protocol
3
+ class AddOffsetsToTxnResponse
4
+
5
+ attr_reader :error_code
6
+
7
+ def initialize(error_code:)
8
+ @error_code = error_code
9
+ end
10
+
11
+ def self.decode(decoder)
12
+ _throttle_time_ms = decoder.int32
13
+ error_code = decoder.int16
14
+ new(error_code: error_code)
15
+ end
16
+
17
+ end
18
+ end
19
+ end
@@ -27,6 +27,8 @@ module Kafka
27
27
  # Value => bytes
28
28
  #
29
29
  class ProduceRequest
30
+ API_MIN_VERSION = 3
31
+
30
32
  attr_reader :transactional_id, :required_acks, :timeout, :messages_for_topics, :compressor
31
33
 
32
34
  # @param required_acks [Integer]
@@ -45,7 +47,7 @@ module Kafka
45
47
  end
46
48
 
47
49
  def api_version
48
- 3
50
+ compressor.codec.nil? ? API_MIN_VERSION : [compressor.codec.produce_api_min_version, API_MIN_VERSION].max
49
51
  end
50
52
 
51
53
  def response_class
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kafka
4
+ module Protocol
5
+ class TxnOffsetCommitRequest
6
+
7
+ def api_key
8
+ TXN_OFFSET_COMMIT_API
9
+ end
10
+
11
+ def api_version
12
+ 2
13
+ end
14
+
15
+ def response_class
16
+ TxnOffsetCommitResponse
17
+ end
18
+
19
+ def initialize(transactional_id:, group_id:, producer_id:, producer_epoch:, offsets:)
20
+ @transactional_id = transactional_id
21
+ @producer_id = producer_id
22
+ @producer_epoch = producer_epoch
23
+ @group_id = group_id
24
+ @offsets = offsets
25
+ end
26
+
27
+ def encode(encoder)
28
+ encoder.write_string(@transactional_id.to_s)
29
+ encoder.write_string(@group_id)
30
+ encoder.write_int64(@producer_id)
31
+ encoder.write_int16(@producer_epoch)
32
+
33
+ encoder.write_array(@offsets) do |topic, partitions|
34
+ encoder.write_string(topic)
35
+ encoder.write_array(partitions) do |partition, offset|
36
+ encoder.write_int32(partition)
37
+ encoder.write_int64(offset[:offset])
38
+ encoder.write_string(nil) # metadata
39
+ encoder.write_int32(offset[:leader_epoch])
40
+ end
41
+ end
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,18 @@
1
+ module Kafka
2
+ module Protocol
3
+ class TxnOffsetCommitResponse
4
+
5
+ attr_reader :error_code
6
+
7
+ def initialize(error_code:)
8
+ @error_code = error_code
9
+ end
10
+
11
+ def self.decode(decoder)
12
+ _throttle_time_ms = decoder.int32
13
+ error_code = decoder.int16
14
+ new(error_code: error_code)
15
+ end
16
+ end
17
+ end
18
+ end
@@ -6,6 +6,10 @@ module Kafka
6
6
  2
7
7
  end
8
8
 
9
+ def produce_api_min_version
10
+ 0
11
+ end
12
+
9
13
  def load
10
14
  require "snappy"
11
15
  rescue LoadError
@@ -54,6 +54,8 @@ module Kafka
54
54
  store.set_default_paths
55
55
  end
56
56
  ssl_context.cert_store = store
57
+ ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
58
+ ssl_context.verify_hostname = true
57
59
  end
58
60
 
59
61
  ssl_context
@@ -57,6 +57,7 @@ module Kafka
57
57
 
58
58
  # once that's connected, we can start initiating the ssl socket
59
59
  @ssl_socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, ssl_context)
60
+ @ssl_socket.hostname = host
60
61
 
61
62
  begin
62
63
  # Initiate the socket connection in the background. If it doesn't fail
@@ -217,6 +217,31 @@ module Kafka
217
217
  raise
218
218
  end
219
219
 
220
+ def send_offsets_to_txn(offsets:, group_id:)
221
+ force_transactional!
222
+
223
+ unless @transaction_state.in_transaction?
224
+ raise 'Transaction is not valid to send offsets'
225
+ end
226
+
227
+ add_response = transaction_coordinator.add_offsets_to_txn(
228
+ transactional_id: @transactional_id,
229
+ producer_id: @producer_id,
230
+ producer_epoch: @producer_epoch,
231
+ group_id: group_id
232
+ )
233
+ Protocol.handle_error(add_response.error_code)
234
+
235
+ send_response = transaction_coordinator.txn_offset_commit(
236
+ transactional_id: @transactional_id,
237
+ group_id: group_id,
238
+ producer_id: @producer_id,
239
+ producer_epoch: @producer_epoch,
240
+ offsets: offsets
241
+ )
242
+ Protocol.handle_error(send_response.error_code)
243
+ end
244
+
220
245
  def in_transaction?
221
246
  @transaction_state.in_transaction?
222
247
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kafka
4
- VERSION = "0.7.6"
4
+ VERSION = "0.7.7"
5
5
  end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kafka
4
+ class ZstdCodec
5
+ def codec_id
6
+ 4
7
+ end
8
+
9
+ def produce_api_min_version
10
+ 7
11
+ end
12
+
13
+ def load
14
+ require "zstd-ruby"
15
+ rescue LoadError
16
+ raise LoadError, "using zstd compression requires adding a dependency on the `zstd-ruby` gem to your Gemfile."
17
+ end
18
+
19
+ def compress(data)
20
+ Zstd.compress(data)
21
+ end
22
+
23
+ def decompress(data)
24
+ Zstd.decompress(data)
25
+ end
26
+ end
27
+ end
@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
39
39
  spec.add_development_dependency "activesupport"
40
40
  spec.add_development_dependency "snappy"
41
41
  spec.add_development_dependency "extlz4"
42
+ spec.add_development_dependency "zstd-ruby"
42
43
  spec.add_development_dependency "colored"
43
44
  spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
44
45
  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.6
4
+ version: 0.7.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-03-14 00:00:00.000000000 Z
11
+ date: 2019-05-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: digest-crc
@@ -164,6 +164,20 @@ dependencies:
164
164
  - - ">="
165
165
  - !ruby/object:Gem::Version
166
166
  version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: zstd-ruby
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
167
181
  - !ruby/object:Gem::Dependency
168
182
  name: colored
169
183
  requirement: !ruby/object:Gem::Requirement
@@ -361,6 +375,8 @@ files:
361
375
  - lib/kafka/produce_operation.rb
362
376
  - lib/kafka/producer.rb
363
377
  - lib/kafka/protocol.rb
378
+ - lib/kafka/protocol/add_offsets_to_txn_request.rb
379
+ - lib/kafka/protocol/add_offsets_to_txn_response.rb
364
380
  - lib/kafka/protocol/add_partitions_to_txn_request.rb
365
381
  - lib/kafka/protocol/add_partitions_to_txn_response.rb
366
382
  - lib/kafka/protocol/alter_configs_request.rb
@@ -416,6 +432,8 @@ files:
416
432
  - lib/kafka/protocol/sasl_handshake_response.rb
417
433
  - lib/kafka/protocol/sync_group_request.rb
418
434
  - lib/kafka/protocol/sync_group_response.rb
435
+ - lib/kafka/protocol/txn_offset_commit_request.rb
436
+ - lib/kafka/protocol/txn_offset_commit_response.rb
419
437
  - lib/kafka/round_robin_assignment_strategy.rb
420
438
  - lib/kafka/sasl/gssapi.rb
421
439
  - lib/kafka/sasl/oauth.rb
@@ -431,6 +449,7 @@ files:
431
449
  - lib/kafka/transaction_manager.rb
432
450
  - lib/kafka/transaction_state_machine.rb
433
451
  - lib/kafka/version.rb
452
+ - lib/kafka/zstd_codec.rb
434
453
  - lib/ruby-kafka.rb
435
454
  - ruby-kafka.gemspec
436
455
  homepage: https://github.com/zendesk/ruby-kafka