ruby-kafka 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08731d84b1c82d5bf46625993d552e80face5fa5dbf71af68ff60cb454f740bf'
-  data.tar.gz: 64aaa4a1b1dcd4200e058bfa1157d07749044752362ff296b5eadd6108c365da
+  metadata.gz: fe93873554ad9e17df699fa49612e0dae938a498c086ae765bf83b79f4a8106f
+  data.tar.gz: eb56576533a3b5d1983e755fc0399464315a47f43d6fc68d9ef20c4df5f6d351
 SHA512:
-  metadata.gz: d154e9ecbbea7014d33ef93b6e21b1897df7628abb51a04094c2209518b43097414b54f03175fa0d59f89368ca2821559b9739c054503046ea74ef6cd875a4d6
-  data.tar.gz: 23680e5dadc11ae5a810708a8d2d7de8e5dbeb0f964164b2d29288737bcd5fa4c9885d10b9660a538a6b9faa27d1accf83af44dabce4ef6d2c56f6b105bb9c91
+  metadata.gz: 19158030f7e028cb5dc350a405867273dadfaaa81208f4684c8c6a68b4d4b2507f97ec13b171e58da1b0277fa6712ccb3db50735cd0db5dd3d9839c0ddcba18b
+  data.tar.gz: d37f2e3ceb765ce5bf267ddcdbcf524251de9356b4e0fa1d16562cf762b99047146c2cda80482792b3f4845b789a8cf8446613f9e49990a22cf73844b9380714

data/CHANGELOG.md

@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## 0.7.7
+- Producer send offsets in transaction (#723)
+- Support zstd compression (#724)
+- Verify SSL Certificates (#730)
+
 ## 0.7.6
 - Introduce regex matching in `Consumer#subscribe` (#700)
 - Only rejoin group on error if we're not in shutdown mode (#711)

data/README.md

@@ -424,6 +424,7 @@ Compression is enabled by passing the `compression_codec` parameter to `#produce
 * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
 * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
 * `:lz4` for [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) compression.
+* `:zstd` for [zstd](https://facebook.github.io/zstd/) compression.
 
 By default, all message sets will be compressed if you specify a compression codec. To increase the compression threshold, set `compression_threshold` to an integer value higher than one.
 

data/lib/kafka/broker.rb

@@ -182,6 +182,18 @@ module Kafka
       send_request(request)
     end
 
+    def add_offsets_to_txn(**options)
+      request = Protocol::AddOffsetsToTxnRequest.new(**options)
+
+      send_request(request)
+    end
+
+    def txn_offset_commit(**options)
+      request = Protocol::TxnOffsetCommitRequest.new(**options)
+
+      send_request(request)
+    end
+
     private
 
     def send_request(request)

data/lib/kafka/client.rb

@@ -233,8 +233,8 @@ module Kafka
     #   result in {BufferOverflow} being raised.
     #
     # @param compression_codec [Symbol, nil] the name of the compression codec to
-    #   use, or nil if no compression should be performed. Valid codecs: `:snappy`
-    #   and `:gzip`.
+    #   use, or nil if no compression should be performed. Valid codecs: `:snappy`,
+    #   `:gzip`, `:lz4`, `:zstd`
     #
     # @param compression_threshold [Integer] the number of messages that needs to
     #   be in a message set before it should be compressed. Note that message sets

data/lib/kafka/compression.rb

@@ -3,27 +3,27 @@
 require "kafka/snappy_codec"
 require "kafka/gzip_codec"
 require "kafka/lz4_codec"
+require "kafka/zstd_codec"
 
 module Kafka
   module Compression
-    CODEC_NAMES = {
-      1 => :gzip,
-      2 => :snappy,
-      3 => :lz4,
-    }.freeze
-
-    CODECS = {
+    CODECS_BY_NAME = {
       :gzip => GzipCodec.new,
       :snappy => SnappyCodec.new,
       :lz4 => LZ4Codec.new,
+      :zstd => ZstdCodec.new,
     }.freeze
 
+    CODECS_BY_ID = CODECS_BY_NAME.each_with_object({}) do |(_, codec), hash|
+      hash[codec.codec_id] = codec
+    end.freeze
+
     def self.codecs
-      CODECS.keys
+      CODECS_BY_NAME.keys
     end
 
     def self.find_codec(name)
-      codec = CODECS.fetch(name) do
+      codec = CODECS_BY_NAME.fetch(name) do
         raise "Unknown compression codec #{name}"
       end
 
@@ -33,11 +33,13 @@ module Kafka
     end
 
     def self.find_codec_by_id(codec_id)
-      codec_name = CODEC_NAMES.fetch(codec_id) do
+      codec = CODECS_BY_ID.fetch(codec_id) do
         raise "Unknown codec id #{codec_id}"
       end
 
-      find_codec(codec_name)
+      codec.load
+
+      codec
     end
   end
 end

data/lib/kafka/compressor.rb

@@ -18,6 +18,7 @@ module Kafka
   # * `compressed_bytesize` – the byte size of the compressed data.
   #
   class Compressor
+    attr_reader :codec
 
     # @param codec_name [Symbol, nil]
     # @param threshold [Integer] the minimum number of messages in a message set

data/lib/kafka/fetched_batch.rb

@@ -13,18 +13,22 @@ module Kafka
     # @return [Integer]
     attr_reader :last_offset
 
+    # @return [Integer]
+    attr_reader :leader_epoch
+
     # @return [Integer] the offset of the most recent message in the partition.
     attr_reader :highwater_mark_offset
 
     # @return [Array<Kafka::FetchedMessage>]
     attr_accessor :messages
 
-    def initialize(topic:, partition:, highwater_mark_offset:, messages:, last_offset: nil)
+    def initialize(topic:, partition:, highwater_mark_offset:, messages:, last_offset: nil, leader_epoch: nil)
       @topic = topic
       @partition = partition
       @highwater_mark_offset = highwater_mark_offset
       @messages = messages
       @last_offset = last_offset
+      @leader_epoch = leader_epoch
     end
 
     def empty?

data/lib/kafka/fetched_batch_generator.rb

@@ -62,11 +62,13 @@ module Kafka
     def extract_records
       records = []
       last_offset = nil
+      leader_epoch = nil
       aborted_transactions = @fetched_partition.aborted_transactions.sort_by(&:first_offset)
       aborted_producer_ids = {}
 
       @fetched_partition.messages.each do |record_batch|
         last_offset = record_batch.last_offset if last_offset.nil? || last_offset < record_batch.last_offset
+        leader_epoch = record_batch.partition_leader_epoch if leader_epoch.nil? || leader_epoch < record_batch.partition_leader_epoch
         # Find the list of aborted producer IDs less than current offset
         unless aborted_transactions.empty?
           if aborted_transactions.first.first_offset <= record_batch.last_offset
@@ -99,6 +101,7 @@ module Kafka
         topic: @topic,
         partition: @fetched_partition.partition,
         last_offset: last_offset,
+        leader_epoch: leader_epoch,
         highwater_mark_offset: @fetched_partition.highwater_mark_offset,
         messages: records
       )

data/lib/kafka/fetched_message.rb

@@ -43,5 +43,6 @@ module Kafka
     def is_control_record
       @message.is_control_record
     end
+
   end
 end

data/lib/kafka/gzip_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       1
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "zlib"
     end

data/lib/kafka/lz4_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       3
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "extlz4"
     rescue LoadError

data/lib/kafka/producer.rb

@@ -68,6 +68,8 @@ module Kafka
   #
   # * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
   # * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
+  # * `:lz4` for [LZ4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) compression.
+  # * `:zstd` for [zstd](https://facebook.github.io/zstd/) compression.
   #
   # By default, all message sets will be compressed if you specify a compression
   # codec. To increase the compression threshold, set `compression_threshold` to
@@ -328,6 +330,20 @@ module Kafka
       @transaction_manager.abort_transaction
     end
 
+    # Sends batch last offset to the consumer group coordinator, and also marks
+    # this offset as part of the current transaction. This offset will be considered
+    # committed only if the transaction is committed successfully.
+    #
+    # This method should be used when you need to batch consumed and produced messages
+    # together, typically in a consume-transform-produce pattern. Thus, the specified
+    # group_id should be the same as config parameter group_id of the used
+    # consumer.
+    #
+    # @return [nil]
+    def send_offsets_to_transaction(batch:, group_id:)
+      @transaction_manager.send_offsets_to_txn(offsets: { batch.topic => { batch.partition => { offset: batch.last_offset + 1, leader_epoch: batch.leader_epoch } } }, group_id: group_id)
+    end
+
     # Syntactic sugar to enable easier transaction usage. Do the following steps
     #
     # - Start the transaction (with Producer#begin_transaction)

data/lib/kafka/protocol.rb

@@ -33,7 +33,9 @@ module Kafka
     DELETE_TOPICS_API         = 20
     INIT_PRODUCER_ID_API      = 22
     ADD_PARTITIONS_TO_TXN_API = 24
+    ADD_OFFSETS_TO_TXN_API    = 25
     END_TXN_API               = 26
+    TXN_OFFSET_COMMIT_API     = 28
     DESCRIBE_CONFIGS_API      = 32
     ALTER_CONFIGS_API         = 33
     CREATE_PARTITIONS_API     = 37
@@ -57,7 +59,9 @@ module Kafka
       DELETE_TOPICS_API         => :delete_topics,
       INIT_PRODUCER_ID_API      => :init_producer_id_api,
       ADD_PARTITIONS_TO_TXN_API => :add_partitions_to_txn_api,
+      ADD_OFFSETS_TO_TXN_API    => :add_offsets_to_txn_api,
       END_TXN_API               => :end_txn_api,
+      TXN_OFFSET_COMMIT_API     => :txn_offset_commit_api,
       DESCRIBE_CONFIGS_API      => :describe_configs_api,
       CREATE_PARTITIONS_API     => :create_partitions
     }
@@ -177,6 +181,10 @@ require "kafka/protocol/fetch_request"
 require "kafka/protocol/fetch_response"
 require "kafka/protocol/list_offset_request"
 require "kafka/protocol/list_offset_response"
+require "kafka/protocol/add_offsets_to_txn_request"
+require "kafka/protocol/add_offsets_to_txn_response"
+require "kafka/protocol/txn_offset_commit_request"
+require "kafka/protocol/txn_offset_commit_response"
 require "kafka/protocol/find_coordinator_request"
 require "kafka/protocol/find_coordinator_response"
 require "kafka/protocol/join_group_request"

data/lib/kafka/protocol/add_offsets_to_txn_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnRequest
+      def initialize(transactional_id: nil, producer_id:, producer_epoch:, group_id:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @group_id = group_id
+      end
+
+      def api_key
+        ADD_OFFSETS_TO_TXN_API
+      end
+
+      def response_class
+        AddOffsetsToTxnResponse
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+        encoder.write_string(@group_id)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/add_offsets_to_txn_response.rb

@@ -0,0 +1,19 @@
+module Kafka
+  module Protocol
+    class AddOffsetsToTxnResponse
+
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+
+    end
+  end
+end

data/lib/kafka/protocol/produce_request.rb

@@ -27,6 +27,8 @@ module Kafka
     #         Value => bytes
     #
     class ProduceRequest
+      API_MIN_VERSION = 3
+
       attr_reader :transactional_id, :required_acks, :timeout, :messages_for_topics, :compressor
 
       # @param required_acks [Integer]
@@ -45,7 +47,7 @@ module Kafka
       end
 
       def api_version
-        3
+        compressor.codec.nil? ? API_MIN_VERSION : [compressor.codec.produce_api_min_version, API_MIN_VERSION].max
       end
 
       def response_class

data/lib/kafka/protocol/txn_offset_commit_request.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class TxnOffsetCommitRequest
+
+      def api_key
+        TXN_OFFSET_COMMIT_API
+      end
+
+      def api_version
+        2
+      end
+
+      def response_class
+        TxnOffsetCommitResponse
+      end
+
+      def initialize(transactional_id:, group_id:, producer_id:, producer_epoch:, offsets:)
+        @transactional_id = transactional_id
+        @producer_id = producer_id
+        @producer_epoch = producer_epoch
+        @group_id = group_id
+        @offsets = offsets
+      end
+
+      def encode(encoder)
+        encoder.write_string(@transactional_id.to_s)
+        encoder.write_string(@group_id)
+        encoder.write_int64(@producer_id)
+        encoder.write_int16(@producer_epoch)
+
+        encoder.write_array(@offsets) do |topic, partitions|
+          encoder.write_string(topic)
+          encoder.write_array(partitions) do |partition, offset|
+            encoder.write_int32(partition)
+            encoder.write_int64(offset[:offset])
+            encoder.write_string(nil) # metadata
+            encoder.write_int32(offset[:leader_epoch])
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/kafka/protocol/txn_offset_commit_response.rb

@@ -0,0 +1,18 @@
+module Kafka
+  module Protocol
+    class TxnOffsetCommitResponse
+
+      attr_reader :error_code
+
+      def initialize(error_code:)
+        @error_code = error_code
+      end
+
+      def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
+        error_code = decoder.int16
+        new(error_code: error_code)
+      end
+    end
+  end
+end

data/lib/kafka/snappy_codec.rb

@@ -6,6 +6,10 @@ module Kafka
       2
     end
 
+    def produce_api_min_version
+      0
+    end
+
     def load
       require "snappy"
     rescue LoadError

data/lib/kafka/ssl_context.rb

@@ -54,6 +54,8 @@ module Kafka
           store.set_default_paths
         end
         ssl_context.cert_store = store
+        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ssl_context.verify_hostname = true
       end
 
       ssl_context

data/lib/kafka/ssl_socket_with_timeout.rb

@@ -57,6 +57,7 @@ module Kafka
 
       # once that's connected, we can start initiating the ssl socket
       @ssl_socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, ssl_context)
+      @ssl_socket.hostname = host
 
       begin
         # Initiate the socket connection in the background. If it doesn't fail

data/lib/kafka/transaction_manager.rb

@@ -217,6 +217,31 @@ module Kafka
       raise
     end
 
+    def send_offsets_to_txn(offsets:, group_id:)
+      force_transactional!
+
+      unless @transaction_state.in_transaction?
+        raise 'Transaction is not valid to send offsets'
+      end
+
+      add_response = transaction_coordinator.add_offsets_to_txn(
+        transactional_id: @transactional_id,
+        producer_id: @producer_id,
+        producer_epoch: @producer_epoch,
+        group_id: group_id
+      )
+      Protocol.handle_error(add_response.error_code)
+
+      send_response = transaction_coordinator.txn_offset_commit(
+        transactional_id: @transactional_id,
+        group_id: group_id,
+        producer_id: @producer_id,
+        producer_epoch: @producer_epoch,
+        offsets: offsets
+      )
+      Protocol.handle_error(send_response.error_code)
+    end
+
     def in_transaction?
       @transaction_state.in_transaction?
     end

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.6"
+  VERSION = "0.7.7"
 end

data/lib/kafka/zstd_codec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kafka
+  class ZstdCodec
+    def codec_id
+      4
+    end
+
+    def produce_api_min_version
+      7
+    end
+
+    def load
+      require "zstd-ruby"
+    rescue LoadError
+      raise LoadError, "using zstd compression requires adding a dependency on the `zstd-ruby` gem to your Gemfile."
+    end
+
+    def compress(data)
+      Zstd.compress(data)
+    end
+
+    def decompress(data)
+      Zstd.decompress(data)
+    end
+  end
+end

data/ruby-kafka.gemspec

@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activesupport"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "extlz4"
+  spec.add_development_dependency "zstd-ruby"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
   spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-14 00:00:00.000000000 Z
+date: 2019-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: colored
   requirement: !ruby/object:Gem::Requirement
@@ -361,6 +375,8 @@ files:
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb
+- lib/kafka/protocol/add_offsets_to_txn_request.rb
+- lib/kafka/protocol/add_offsets_to_txn_response.rb
 - lib/kafka/protocol/add_partitions_to_txn_request.rb
 - lib/kafka/protocol/add_partitions_to_txn_response.rb
 - lib/kafka/protocol/alter_configs_request.rb
@@ -416,6 +432,8 @@ files:
 - lib/kafka/protocol/sasl_handshake_response.rb
 - lib/kafka/protocol/sync_group_request.rb
 - lib/kafka/protocol/sync_group_response.rb
+- lib/kafka/protocol/txn_offset_commit_request.rb
+- lib/kafka/protocol/txn_offset_commit_response.rb
 - lib/kafka/round_robin_assignment_strategy.rb
 - lib/kafka/sasl/gssapi.rb
 - lib/kafka/sasl/oauth.rb
@@ -431,6 +449,7 @@ files:
 - lib/kafka/transaction_manager.rb
 - lib/kafka/transaction_state_machine.rb
 - lib/kafka/version.rb
+- lib/kafka/zstd_codec.rb
 - lib/ruby-kafka.rb
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka