ruby-kafka 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8ea1d759bb30b26f2130fd560d2f09bd46b0960add9a1e65460e9643cb50142
-  data.tar.gz: 9816bf4c2c73567aa0a12fbc0cfe79af84861517a1845eaba5881ac05bd99f77
+  metadata.gz: 5f4441fd6947e65aaf63f57a7248000af3b1140eca6b33ed128cc51f24fa415b
+  data.tar.gz: ea4de0c4d19991767378cc4672bd7da47a3847b2931edd900d3d2f8e463bca0a
 SHA512:
-  metadata.gz: c53dbab79d1157151328c2d718d1d2d2ba9600208419f5788c231df3cc63542f05bfe514cea199d72dde781eab252cf89fd9889f0e22c148afa210d1f499e28a
-  data.tar.gz: 8a8bc9e3dc7d7edb465ebe990c4343330b03db21d595f93cb82698e7abe68166f132975e3e1ff95805b663c2d8227b87e74342b4efab412fda3c4d43be190a97
+  metadata.gz: ffd480a08e25377139e65aab9aeb0d8d85056f033142104ae6593b7b39bafbd9db0de3b4487224a608550b567b072f32b30e6d61c2f2bbcfdc47423d3aece3f8
+  data.tar.gz: b0acbf1f1024d50d1169a68316a67041a837965a76d79c0143228b5e142cdd412b4b9813257e6a40a400bf5a89a1fb70f246932ffb9f280e243848a73a66653c

data/CHANGELOG.md

@@ -4,6 +4,14 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## 0.7.4
+- Fix wrong encoding calculation that leads to message corruption (#682, #680).
+- Change the log level of the 'Committing offsets' message to debug (#640).
+- Avoid Ruby warnings about unused vars (#679).
+- Synchronously commit offsets after HeartbeatError (#676).
+- Discard messages that were fetched under a previous consumer group generation (#665).
+- Support specifying an ssl client certificates key passphrase (#667).
+
 ## 0.7.3
 
 - Synchronize access to @worker_thread and @timer_thread in AsyncProducer to prevent creating multiple threads (#661).

data/README.md

@@ -921,6 +921,7 @@ kafka = Kafka.new(
   ssl_ca_cert: File.read('my_ca_cert.pem'),
   ssl_client_cert: File.read('my_client_cert.pem'),
   ssl_client_cert_key: File.read('my_client_cert_key.pem'),
+  ssl_client_cert_key_password: 'my_client_cert_key_password',
   # ...
 )
 ```

data/lib/kafka.rb

@@ -19,81 +19,102 @@ module Kafka
     end
   end
 
-  # A fetch operation was executed with no partitions specified.
-  class NoPartitionsToFetchFrom < Error
-  end
-
-  # A message in a partition is larger than the maximum we've asked for.
-  class MessageTooLargeToRead < Error
-  end
-
-  # A connection has been unused for too long, we assume the server has killed it.
-  class IdleConnection < Error
-  end
-
   # Subclasses of this exception class map to an error code described in the
   # Kafka protocol specification.
-  #
-  # See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
+  # https://kafka.apache.org/protocol#protocol_error_codes
   class ProtocolError < Error
   end
 
-  # This indicates that a message contents does not match its CRC.
-  class CorruptMessage < ProtocolError
-  end
-
-  # When the record array length doesn't match real number of received records
-  class InsufficientDataMessage < Error
-  end
-
+  # -1
+  # The server experienced an unexpected error when processing the request
   class UnknownError < ProtocolError
   end
 
+  # 1
+  # The requested offset is not within the range of offsets maintained by the server.
   class OffsetOutOfRange < ProtocolError
     attr_accessor :topic, :partition, :offset
   end
 
+  # 2
+  # This indicates that a message contents does not match its CRC.
+  class CorruptMessage < ProtocolError
+  end
+
+  # 3
   # The request is for a topic or partition that does not exist on the broker.
   class UnknownTopicOrPartition < ProtocolError
   end
 
+  # 4
   # The message has a negative size.
   class InvalidMessageSize < ProtocolError
   end
 
+  # 5
   # This error is thrown if we are in the middle of a leadership election and
   # there is currently no leader for this partition and hence it is unavailable
   # for writes.
   class LeaderNotAvailable < ProtocolError
   end
 
+  # 6
   # This error is thrown if the client attempts to send messages to a replica
   # that is not the leader for some partition. It indicates that the client's
   # metadata is out of date.
   class NotLeaderForPartition < ProtocolError
   end
 
+  # 7
   # This error is thrown if the request exceeds the user-specified time limit
   # in the request.
   class RequestTimedOut < ProtocolError
   end
 
+  # 8
+  # The broker is not available.
   class BrokerNotAvailable < ProtocolError
   end
 
+  # 9
+  # Raised if a replica is expected on a broker, but is not. Can be safely ignored.
+  class ReplicaNotAvailable < ProtocolError
+  end
+
+  # 10
   # The server has a configurable maximum message size to avoid unbounded memory
   # allocation. This error is thrown if the client attempt to produce a message
   # larger than this maximum.
   class MessageSizeTooLarge < ProtocolError
   end
 
+  # 11
+  # The controller moved to another broker.
+  class StaleControllerEpoch < ProtocolError
+  end
+
+  # 12
   # If you specify a string larger than configured maximum for offset metadata.
   class OffsetMetadataTooLarge < ProtocolError
   end
 
+  # 13
+  # The server disconnected before a response was received.
+  class NetworkException < ProtocolError
+  end
+
+  # 14
+  # The coordinator is loading and hence can't process requests.
+  class CoordinatorLoadInProgress < ProtocolError
+  end
+
+  # 15
+  # The coordinator is not available.
   class CoordinatorNotAvailable < ProtocolError
   end
 
+  # 16
+  # This is not the correct coordinator.
   class NotCoordinatorForGroup < ProtocolError
   end
 
@@ -104,6 +125,7 @@ module Kafka
   class InvalidTopic < ProtocolError
   end
 
+  # 18
   # If a message batch in a produce request exceeds the maximum configured
   # segment size.
   class RecordListTooLarge < ProtocolError
@@ -127,28 +149,38 @@ module Kafka
   class InvalidRequiredAcks < ProtocolError
   end
 
-  # 9
-  # Raised if a replica is expected on a broker, but is not. Can be safely ignored.
-  class ReplicaNotAvailable < ProtocolError
+  # 22
+  # Specified group generation id is not valid.
+  class IllegalGeneration < ProtocolError
   end
 
-  # 25
-  class UnknownMemberId < ProtocolError
+  # 23
+  # The group member's supported protocols are incompatible with those of existing members or first group member tried to join with empty protocol type or empty protocol list.
+  class InconsistentGroupProtocol < ProtocolError
   end
 
-  # 27
-  class RebalanceInProgress < ProtocolError
+  # 24
+  # The configured groupId is invalid
+  class InvalidGroupId < ProtocolError
   end
 
-  # 22
-  class IllegalGeneration < ProtocolError
+  # 25
+  # The coordinator is not aware of this member.
+  class UnknownMemberId < ProtocolError
   end
 
   # 26
+  # The session timeout is not within the range allowed by the broker
   class InvalidSessionTimeout < ProtocolError
   end
 
+  # 27
+  # The group is rebalancing, so a rejoin is needed.
+  class RebalanceInProgress < ProtocolError
+  end
+
   # 28
+  # The committing offset data size is not valid
   class InvalidCommitOffsetSize < ProtocolError
   end
 
@@ -165,10 +197,12 @@ module Kafka
   end
 
   # 32
+  # The timestamp of the message is out of acceptable range.
   class InvalidTimestamp < ProtocolError
   end
 
   # 33
+  # The broker does not support the requested SASL mechanism.
   class UnsupportedSaslMechanism < ProtocolError
   end
 
@@ -185,10 +219,12 @@ module Kafka
   end
 
   # 37
+  # Number of partitions is below 1.
   class InvalidPartitions < ProtocolError
   end
 
   # 38
+  # Replication factor is below 1 or larger than the number of available brokers.
   class InvalidReplicationFactor < ProtocolError
   end
 
@@ -201,6 +237,7 @@ module Kafka
   end
 
   # 41
+  # This is not the correct controller for this cluster.
   class NotController < ProtocolError
   end
 
@@ -208,6 +245,75 @@ module Kafka
   class InvalidRequest < ProtocolError
   end
 
+  # 43
+  # The message format version on the broker does not support the request.
+  class UnsupportedForMessageFormat < ProtocolError
+  end
+
+  # 44
+  # Request parameters do not satisfy the configured policy.
+  class PolicyViolation < ProtocolError
+  end
+
+  # 45
+  # The broker received an out of order sequence number
+  class OutOfOrderSequenceNumberError < Error
+  end
+
+  # 46
+  # The broker received a duplicate sequence number
+  class DuplicateSequenceNumberError < Error
+  end
+
+  # 47
+  # Producer attempted an operation with an old epoch. Either there is a newer producer with the same transactionalId, or the producer's transaction has been expired by the broker.
+  class InvalidProducerEpochError < Error
+  end
+
+  # 48
+  # The producer attempted a transactional operation in an invalid state
+  class InvalidTxnStateError < Error
+  end
+
+  # 49
+  # The producer attempted to use a producer id which is not currently assigned to its transactional id
+  class InvalidProducerIDMappingError < Error
+  end
+
+  # 50
+  # The transaction timeout is larger than the maximum value allowed by the broker (as configured by transaction.max.timeout.ms).
+  class InvalidTransactionTimeoutError < Error
+  end
+
+  # 51
+  # The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing
+  class ConcurrentTransactionError < Error
+  end
+
+  # 52
+  # Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer
+  class TransactionCoordinatorFencedError < Error
+  end
+
+  ###
+  # ruby-kafka errors
+  ###
+
+  # A fetch operation was executed with no partitions specified.
+  class NoPartitionsToFetchFrom < Error
+  end
+
+  # A message in a partition is larger than the maximum we've asked for.
+  class MessageTooLargeToRead < Error
+  end
+
+  # A connection has been unused for too long, we assume the server has killed it.
+  class IdleConnection < Error
+  end
+
+  # When the record array length doesn't match real number of received records
+  class InsufficientDataMessage < Error
+  end
   # Raised when there's a network connection error.
   class ConnectionError < Error
   end
@@ -245,38 +351,6 @@ module Kafka
   class FailedScramAuthentication < SaslScramError
   end
 
-  # The broker received an out of order sequence number
-  class OutOfOrderSequenceNumberError < Error
-  end
-
-  # The broker received a duplicate sequence number
-  class DuplicateSequenceNumberError < Error
-  end
-
-  # Producer attempted an operation with an old epoch. Either there is a newer producer with the same transactionalId, or the producer's transaction has been expired by the broker.
-  class InvalidProducerEpochError < Error
-  end
-
-  # The producer attempted a transactional operation in an invalid state
-  class InvalidTxnStateError < Error
-  end
-
-  # The producer attempted to use a producer id which is not currently assigned to its transactional id
-  class InvalidProducerIDMappingError < Error
-  end
-
-  # The transaction timeout is larger than the maximum value allowed by the broker (as configured by transaction.max.timeout.ms).
-  class InvalidTransactionTimeoutError < Error
-  end
-
-  # The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing
-  class ConcurrentTransactionError < Error
-  end
-
-  # Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer
-  class TransactionCoordinatorFencedError < Error
-  end
-
   # Initializes a new Kafka client.
   #
   # @see Client#initialize

data/lib/kafka/client.rb

@@ -46,6 +46,9 @@ module Kafka
     # @param ssl_client_cert_key [String, nil] a PEM encoded client cert key to use with an
     #   SSL connection. Must be used in combination with ssl_client_cert.
     #
+    # @param ssl_client_cert_key_password [String, nil] the password required to read the
+    #   ssl_client_cert_key. Must be used in combination with ssl_client_cert_key.
+    #
     # @param sasl_gssapi_principal [String, nil] a KRB5 principal
     #
     # @param sasl_gssapi_keytab [String, nil] a KRB5 keytab filepath
@@ -61,8 +64,8 @@ module Kafka
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
-                   ssl_client_cert_chain: nil, sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
-                   sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
+                   ssl_client_cert_key_password: nil, ssl_client_cert_chain: nil, sasl_gssapi_principal: nil,
+                   sasl_gssapi_keytab: nil, sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
                    sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
                    sasl_over_ssl: true, ssl_ca_certs_from_system: false)
       @logger = logger || Logger.new(nil)
@@ -74,6 +77,7 @@ module Kafka
         ca_cert: ssl_ca_cert,
         client_cert: ssl_client_cert,
         client_cert_key: ssl_client_cert_key,
+        client_cert_key_password: ssl_client_cert_key_password,
         client_cert_chain: ssl_client_cert_chain,
         ca_certs_from_system: ssl_ca_certs_from_system,
       )

data/lib/kafka/consumer.rb

@@ -394,7 +394,10 @@ module Kafka
           @instrumenter.instrument("loop.consumer") do
             yield
           end
-        rescue HeartbeatError, OffsetCommitError
+        rescue HeartbeatError
+          make_final_offsets_commit!
+          join_group
+        rescue OffsetCommitError
           join_group
         rescue RebalanceInProgress
           @logger.warn "Group rebalance in progress, re-joining..."

data/lib/kafka/fetcher.rb

@@ -28,6 +28,10 @@ module Kafka
 
       # The maximum number of bytes to fetch per partition, by topic.
       @max_bytes_per_partition = {}
+
+      # An incrementing counter used to synchronize resets between the
+      # foreground and background thread.
+      @current_reset_counter = 0
     end
 
     def subscribe(topic, max_bytes_per_partition:)
@@ -62,7 +66,8 @@ module Kafka
     end
 
     def reset
-      @commands << [:reset, []]
+      @current_reset_counter = current_reset_counter + 1
+      @commands << [:reset]
     end
 
     def data?
@@ -70,11 +75,23 @@ module Kafka
     end
 
     def poll
-      @queue.deq
+      tag, message, reset_counter = @queue.deq
+
+      # Batches are tagged with the current reset counter value. If the batch
+      # has a reset_counter < current_reset_counter, we know it was fetched
+      # prior to the most recent reset and should be discarded.
+      if tag == :batches && message.any? && current_reset_counter > reset_counter
+        @logger.warn "Skipping stale messages buffered prior to reset"
+        return tag, []
+      end
+
+      return [tag, message]
     end
 
     private
 
+    attr_reader :current_reset_counter
+
     def loop
       @instrumenter.instrument("loop.fetcher", {
         queue_size: @queue.size,
@@ -149,7 +166,7 @@ module Kafka
         @next_offsets[batch.topic][batch.partition] = batch.last_offset + 1 unless batch.unknown_last_offset?
       end
 
-      @queue << [:batches, batches]
+      @queue << [:batches, batches, current_reset_counter]
     rescue Kafka::NoPartitionsToFetchFrom
       @logger.warn "No partitions to fetch from, sleeping for 1s"
       sleep 1

data/lib/kafka/offset_manager.rb

@@ -120,7 +120,7 @@ module Kafka
     def commit_offsets(recommit = false)
       offsets = offsets_to_commit(recommit)
       unless offsets.empty?
-        @logger.info "Committing offsets#{recommit ? ' with recommit' : ''}: #{prettify_offsets(offsets)}"
+        @logger.debug "Committing offsets#{recommit ? ' with recommit' : ''}: #{prettify_offsets(offsets)}"
 
         @group.commit_offsets(offsets)
 

data/lib/kafka/protocol.rb

@@ -75,7 +75,10 @@ module Kafka
        8 => BrokerNotAvailable,
        9 => ReplicaNotAvailable,
       10 => MessageSizeTooLarge,
+      11 => StaleControllerEpoch,
       12 => OffsetMetadataTooLarge,
+      13 => NetworkException,
+      14 => CoordinatorLoadInProgress,
       15 => CoordinatorNotAvailable,
       16 => NotCoordinatorForGroup,
       17 => InvalidTopic,
@@ -84,6 +87,8 @@ module Kafka
       20 => NotEnoughReplicasAfterAppend,
       21 => InvalidRequiredAcks,
       22 => IllegalGeneration,
+      23 => InconsistentGroupProtocol,
+      24 => InvalidGroupId,
       25 => UnknownMemberId,
       26 => InvalidSessionTimeout,
       27 => RebalanceInProgress,
@@ -102,6 +107,8 @@ module Kafka
       40 => InvalidConfig,
       41 => NotController,
       42 => InvalidRequest,
+      43 => UnsupportedForMessageFormat,
+      44 => PolicyViolation,
       45 => OutOfOrderSequenceNumberError,
       46 => DuplicateSequenceNumberError,
       47 => InvalidProducerEpochError,

data/lib/kafka/protocol/describe_groups_response.rb

@@ -45,7 +45,7 @@ module Kafka
             member_id = decoder.string
             client_id = decoder.string
             client_host = decoder.string
-            metadata = decoder.bytes
+            _metadata = decoder.bytes
             assignment = MemberAssignment.decode(Decoder.from_string(decoder.bytes))
 
             Member.new(

data/lib/kafka/protocol/encoder.rb

@@ -133,7 +133,7 @@ module Kafka
         int = ~int | 1 if int < 0
 
         chunks = []
-        while int & 0xff80 != 0
+        while int >> 7 != 0
           chunks << (int & 0x7f | 0x80)
           int >>= 7
         end

data/lib/kafka/protocol/message.rb

@@ -67,7 +67,7 @@ module Kafka
         offset = decoder.int64
         message_decoder = Decoder.from_string(decoder.bytes)
 
-        crc = message_decoder.int32
+        _crc = message_decoder.int32
         magic_byte = message_decoder.int8
         attributes = message_decoder.int8
 

data/lib/kafka/protocol/metadata_response.rb

@@ -113,7 +113,7 @@ module Kafka
       # @param node_id [Integer] the node id of the broker.
       # @return [Kafka::BrokerInfo] information about the broker.
       def find_broker(node_id)
-        broker = @brokers.find {|broker| broker.node_id == node_id }
+        broker = @brokers.find {|b| b.node_id == node_id }
 
         raise Kafka::NoSuchBroker, "No broker with id #{node_id}" if broker.nil?
 
@@ -145,7 +145,7 @@ module Kafka
           node_id = decoder.int32
           host = decoder.string
           port = decoder.int32
-          rack = decoder.string
+          _rack = decoder.string
 
           BrokerInfo.new(
             node_id: node_id,
@@ -159,7 +159,7 @@ module Kafka
         topics = decoder.array do
           topic_error_code = decoder.int16
           topic_name = decoder.string
-          is_internal = decoder.boolean
+          _is_internal = decoder.boolean
 
           partitions = decoder.array do
             PartitionMetadata.new(

data/lib/kafka/ssl_context.rb

@@ -6,15 +6,20 @@ module Kafka
   module SslContext
     CLIENT_CERT_DELIMITER = "\n-----END CERTIFICATE-----\n"
 
-    def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_chain: nil, ca_certs_from_system: nil)
-      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || client_cert_chain || ca_certs_from_system
+    def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, client_cert_key_password: nil, client_cert_chain: nil, ca_certs_from_system: nil)
+      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || client_cert_key_password || client_cert_chain || ca_certs_from_system
 
       ssl_context = OpenSSL::SSL::SSLContext.new
 
       if client_cert && client_cert_key
+        if client_cert_key_password
+          cert_key = OpenSSL::PKey.read(client_cert_key, client_cert_key_password)
+        else
+          cert_key = OpenSSL::PKey.read(client_cert_key)
+        end
         context_params = {
           cert: OpenSSL::X509::Certificate.new(client_cert),
-          key: OpenSSL::PKey.read(client_cert_key),
+          key: cert_key
         }
         if client_cert_chain
           certs = []
@@ -33,6 +38,8 @@ module Kafka
         raise ArgumentError, "Kafka client initialized with `ssl_client_cert_chain`, but no `ssl_client_cert`. Please provide cert, key and chain."
       elsif client_cert_chain && !client_cert_key
         raise ArgumentError, "Kafka client initialized with `ssl_client_cert_chain`, but no `ssl_client_cert_key`. Please provide cert, key and chain."
+      elsif client_cert_key_password && !client_cert_key
+        raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key_password`, but no `ssl_client_cert_key`. Please provide both."
       end
 
       if ca_cert || ca_cert_file_path || ca_certs_from_system

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.3"
+  VERSION = "0.7.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-15 00:00:00.000000000 Z
+date: 2018-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc