ruby-kafka 0.7.0.beta1 → 0.7.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1662d1f7534da36a4a6f5ade9e194fd5142034ba4b53d1b6291fcf233e7eb684
-  data.tar.gz: 36b9ecb999f55cfd797c23da795c214c7b9712b42a88e6343024e0e17a123eac
+  metadata.gz: e26d3e4028b3b35f16ab12973d386651580c76258e572621e2a84a337fee31f8
+  data.tar.gz: 27f0a98a1cd057d81443c4b43a71a60a697f5c3add8adf30ca882161da82a242
 SHA512:
-  metadata.gz: 8b22b391b89199f8001401069f5040bac248866f82b8bd1f69ecf3d67d947ef7f54f6d933a68a9114c2a28f54097be4cedb3c74743bae1b231f313a4427ad257
-  data.tar.gz: '0318b5a5cd085ea2c96e7e1bf3c91d914c3898d555b6bdf5681851907e0dec4e2f3ab39c95bf8c289558c41d0c901b243ea7e240e2271235769a2378069e1bbc'
+  metadata.gz: 84690e7b9de72785edb5cb4e8d32e9127b4ea2a8b03fe9d8232455d518818813a270534f3c1b1cd033005adc26a614c0f63ef2190a29da7693e3ef729fb83c02
+  data.tar.gz: 4dc2def15cd9072e201b7a1f012a0cd593a0be76a859b1abacc608580b5257f265e013c6104aee4decf3237daf6522037cb510d834d138e83c44edc15122301c

data/CHANGELOG.md

@@ -7,6 +7,11 @@ Changes and additions to the library will be listed here.
 - Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
 - Support record headers (#604).
 - Add instrumenter and logger when async message delivery fails (#603).
+- Upgrade and rename GroupCoordinator API to FindCoordinator API (#606).
+- Refresh cluster metadata after topic re-assignment (#609).
+- Disable SASL over SSL with a new config (#613).
+- Allow listing brokers in a cluster (#626).
+- Fix Fetcher's message skipping (#625).
 
 ## 0.6.7
 

data/README.md

@@ -771,7 +771,16 @@ All notifications have `group_id` in the payload, referring to the Kafka consume
   
 * `leave_group.consumer.kafka` is sent whenever a consumer leaves a consumer group. It includes the following payload:
   * `group_id` is the consumer group id.
- 
+  
+* `seek.consumer.kafka` is sent when a consumer first seeks to an offset. It includes the following payload:
+  * `group_id` is the consumer group id.
+  * `topic` is the topic we are seeking in.
+  * `partition` is the partition we are seeking in.
+  * `offset` is the offset we have seeked to.
+  
+* `heartbeat.consumer.kafka` is sent when a consumer group completes a heartbeat. It includes the following payload:
+  * `group_id` is the consumer group id.
+  * `topic_partitions` is a hash of { topic_name => array of assigned partition IDs }
   
 #### Connection Notifications
 
@@ -926,7 +935,7 @@ Typically, Kafka certificates come in the JKS format, which isn't supported by r
 
 Kafka has support for using SASL to authenticate clients. Currently GSSAPI, SCRAM and PLAIN mechanisms are supported by ruby-kafka.
 
-**NOTE:** In order to use SASL for authentication, you need to configure SSL encryption by passing `ssl_ca_cert` or enabling `ssl_ca_certs_from_system`.
+**NOTE:** With SASL for authentication, it is highly recommended to use SSL encryption. The default behavior of ruby-kafka enforces you to use SSL and you need to configure SSL encryption by passing `ssl_ca_cert` or enabling `ssl_ca_certs_from_system`. However, this strict SSL mode check can be disabled by setting  `sasl_over_ssl` to `false` while initializing the client.
 
 ##### GSSAPI
 In order to authenticate using GSSAPI, set your principal and optionally your keytab when initializing the Kafka client:
@@ -946,15 +955,13 @@ In order to authenticate using PLAIN, you must set your username and password wh
 ```ruby
 kafka = Kafka.new(
   ["kafka1:9092"],
-  ssl_ca_cert: File.read('/etc/openssl/cert.pem'), # Optional but highly recommended
+  ssl_ca_cert: File.read('/etc/openssl/cert.pem'),
   sasl_plain_username: 'username',
   sasl_plain_password: 'password'
   # ...
 )
 ```
 
-**NOTE**: It is __highly__ recommended that you use SSL for encryption when using SASL_PLAIN
-
 ##### SCRAM
 Since 0.11 kafka supports [SCRAM](https://kafka.apache.org/documentation.html#security_sasl_scram). 
 

data/lib/kafka.rb

@@ -91,7 +91,7 @@ module Kafka
   class OffsetMetadataTooLarge < ProtocolError
   end
 
-  class GroupCoordinatorNotAvailable < ProtocolError
+  class CoordinatorNotAvailable < ProtocolError
   end
 
   class NotCoordinatorForGroup < ProtocolError

data/lib/kafka/broker.rb

@@ -104,8 +104,8 @@ module Kafka
       send_request(request)
     end
 
-    def find_group_coordinator(**options)
-      request = Protocol::GroupCoordinatorRequest.new(**options)
+    def find_coordinator(**options)
+      request = Protocol::FindCoordinatorRequest.new(**options)
 
       send_request(request)
     end

data/lib/kafka/broker_info.rb

@@ -0,0 +1,16 @@
+# Represents a broker in a Kafka cluster.
+module Kafka
+  class BrokerInfo
+    attr_reader :node_id, :host, :port
+
+    def initialize(node_id:, host:, port:)
+      @node_id = node_id
+      @host = host
+      @port = port
+    end
+
+    def to_s
+      "#{host}:#{port} (node_id=#{node_id})"
+    end
+  end
+end

data/lib/kafka/client.rb

@@ -2,6 +2,7 @@
 
 require "kafka/ssl_context"
 require "kafka/cluster"
+require "kafka/broker_info"
 require "kafka/producer"
 require "kafka/consumer"
 require "kafka/heartbeat"
@@ -54,12 +55,15 @@ module Kafka
     #
     # @param sasl_scram_mechanism [String, nil] Scram mechanism, either "sha256" or "sha512"
     #
+    # @param sasl_over_ssl [Boolean] whether to enforce SSL with SASL
+    #
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
                    sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
                    sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
-                   sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil, ssl_ca_certs_from_system: false)
+                   sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
+                   sasl_over_ssl: true, ssl_ca_certs_from_system: false)
       @logger = logger || Logger.new(nil)
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
@@ -84,7 +88,7 @@ module Kafka
         logger: @logger
       )
 
-      if sasl_authenticator.enabled? && ssl_context.nil?
+      if sasl_authenticator.enabled? && sasl_over_ssl && ssl_context.nil?
         raise ArgumentError, "SASL authentication requires that SSL is configured"
       end
 
@@ -313,6 +317,7 @@ module Kafka
 
       fetcher = Fetcher.new(
         cluster: initialize_cluster,
+        group: group,
         logger: @logger,
         instrumenter: instrumenter,
         max_queue_size: fetcher_max_queue_size
@@ -473,7 +478,7 @@ module Kafka
 
         batches.each do |batch|
           batch.messages.each(&block)
-          offsets[batch.partition] = batch.last_offset + 1
+          offsets[batch.partition] = batch.last_offset + 1 unless batch.empty?
         end
       end
     end
@@ -648,6 +653,20 @@ module Kafka
       @cluster.apis
     end
 
+    # List all brokers in the cluster.
+    #
+    # @return [Array<Kafka::BrokerInfo>] the list of brokers.
+    def brokers
+      @cluster.cluster_info.brokers
+    end
+
+    # The current controller broker in the cluster.
+    #
+    # @return [Kafka::BrokerInfo] information on the controller broker.
+    def controller_broker
+      brokers.find {|broker| broker.node_id == @cluster.cluster_info.controller_id }
+    end
+
     # Closes all connections to the Kafka brokers and frees up used resources.
     #
     # @return [nil]

data/lib/kafka/cluster.rb

@@ -40,14 +40,17 @@ module Kafka
     # @param topics [Array<String>]
     # @return [nil]
     def add_target_topics(topics)
-      new_topics = Set.new(topics) - @target_topics
+      topics = Set.new(topics)
+      unless topics.subset?(@target_topics)
+        new_topics = topics - @target_topics
 
-      unless new_topics.empty?
-        @logger.info "New topics added to target list: #{new_topics.to_a.join(', ')}"
+        unless new_topics.empty?
+          @logger.info "New topics added to target list: #{new_topics.to_a.join(', ')}"
 
-        @target_topics.merge(new_topics)
+          @target_topics.merge(new_topics)
 
-        refresh_metadata!
+          refresh_metadata!
+        end
       end
     end
 
@@ -116,9 +119,12 @@ module Kafka
       cluster_info.brokers.each do |broker_info|
         begin
           broker = connect_to_broker(broker_info.node_id)
-          response = broker.find_group_coordinator(group_id: group_id)
+          response = broker.find_coordinator(
+            coordinator_type: Kafka::Protocol::COORDINATOR_TYPE_GROUP,
+            coordinator_key: group_id
+          )
 
-          Protocol.handle_error(response.error_code)
+          Protocol.handle_error(response.error_code, response.error_message)
 
           coordinator_id = response.coordinator_id
 
@@ -137,7 +143,7 @@ module Kafka
           @logger.debug "Connected to coordinator: #{coordinator} for group `#{group_id}`"
 
           return coordinator
-        rescue GroupCoordinatorNotAvailable
+        rescue CoordinatorNotAvailable
           @logger.debug "Coordinator not available; retrying in 1s"
           sleep 1
           retry
@@ -357,16 +363,16 @@ module Kafka
       @broker_pool.close
     end
 
+    def cluster_info
+      @cluster_info ||= fetch_cluster_info
+    end
+
     private
 
     def get_leader_id(topic, partition)
       cluster_info.find_leader_id(topic, partition)
     end
 
-    def cluster_info
-      @cluster_info ||= fetch_cluster_info
-    end
-
     # Fetches the cluster metadata.
     #
     # This is used to update the partition leadership information, among other things.

data/lib/kafka/consumer.rb

@@ -444,6 +444,7 @@ module Kafka
       if old_generation_id && @group.generation_id != old_generation_id + 1
         # We've been out of the group for at least an entire generation, no
         # sense in trying to hold on to offset data
+        clear_current_offsets
         @offset_manager.clear_offsets
       else
         # After rejoining the group we may have been assigned a new set of
@@ -451,6 +452,7 @@ module Kafka
         # having the consumer go back and reprocess messages if it's assigned
         # a partition it used to be assigned to way back. For that reason, we
         # only keep commits for the partitions that we're still assigned.
+        clear_current_offsets(excluding: @group.assigned_partitions)
         @offset_manager.clear_offsets_excluding(@group.assigned_partitions)
       end
 
@@ -537,5 +539,13 @@ module Kafka
     def pause_for(topic, partition)
       @pauses[topic][partition]
     end
+
+    def clear_current_offsets(excluding: {})
+      @current_offsets.each do |topic, partitions|
+        partitions.keep_if do |partition, _|
+          excluding.fetch(topic, []).include?(partition)
+        end
+      end
+    end
   end
 end

data/lib/kafka/consumer_group.rb

@@ -5,7 +5,7 @@ require "kafka/round_robin_assignment_strategy"
 
 module Kafka
   class ConsumerGroup
-    attr_reader :assigned_partitions, :generation_id
+    attr_reader :assigned_partitions, :generation_id, :group_id
 
     def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:, instrumenter:)
       @cluster = cluster
@@ -100,13 +100,18 @@ module Kafka
     def heartbeat
       @logger.debug "Sending heartbeat..."
 
-      response = coordinator.heartbeat(
-        group_id: @group_id,
-        generation_id: @generation_id,
-        member_id: @member_id,
-      )
+      @instrumenter.instrument('heartbeat.consumer',
+                               group_id: @group_id,
+                               topic_partitions: @assigned_partitions) do
 
-      Protocol.handle_error(response.error_code)
+        response = coordinator.heartbeat(
+          group_id: @group_id,
+          generation_id: @generation_id,
+          member_id: @member_id,
+        )
+
+        Protocol.handle_error(response.error_code)
+      end
     rescue ConnectionError, UnknownMemberId, RebalanceInProgress, IllegalGeneration => e
       @logger.error "Error sending heartbeat: #{e}"
       raise HeartbeatError, e
@@ -183,7 +188,7 @@ module Kafka
 
     def coordinator
       @coordinator ||= @cluster.get_group_coordinator(group_id: @group_id)
-    rescue GroupCoordinatorNotAvailable
+    rescue CoordinatorNotAvailable
       @logger.error "Group coordinator not available for group `#{@group_id}`"
 
       sleep 1

data/lib/kafka/fetcher.rb

@@ -6,11 +6,12 @@ module Kafka
   class Fetcher
     attr_reader :queue
 
-    def initialize(cluster:, logger:, instrumenter:, max_queue_size:)
+    def initialize(cluster:, logger:, instrumenter:, max_queue_size:, group:)
       @cluster = cluster
       @logger = logger
       @instrumenter = instrumenter
       @max_queue_size = max_queue_size
+      @group = group
 
       @queue = Queue.new
       @commands = Queue.new
@@ -122,6 +123,11 @@ module Kafka
     end
 
     def handle_seek(topic, partition, offset)
+      @instrumenter.instrument('seek.consumer',
+                               group_id: @group.group_id,
+                               topic: topic,
+                               partition: partition,
+                               offset: offset)
       @logger.info "Seeking #{topic}/#{partition} to offset #{offset}"
       @next_offsets[topic][partition] = offset
     end
@@ -138,9 +144,9 @@ module Kafka
             highwater_mark_offset: batch.highwater_mark_offset,
             message_count: batch.messages.count,
           })
-        end
 
-        @next_offsets[batch.topic][batch.partition] = batch.last_offset + 1
+          @next_offsets[batch.topic][batch.partition] = batch.last_offset + 1
+        end
       end
 
       @queue << [:batches, batches]

data/lib/kafka/protocol.rb

@@ -20,7 +20,7 @@ module Kafka
     TOPIC_METADATA_API = 3
     OFFSET_COMMIT_API = 8
     OFFSET_FETCH_API = 9
-    GROUP_COORDINATOR_API = 10
+    FIND_COORDINATOR_API = 10
     JOIN_GROUP_API = 11
     HEARTBEAT_API = 12
     LEAVE_GROUP_API = 13
@@ -43,7 +43,7 @@ module Kafka
       TOPIC_METADATA_API => :topic_metadata,
       OFFSET_COMMIT_API => :offset_commit,
       OFFSET_FETCH_API => :offset_fetch,
-      GROUP_COORDINATOR_API => :group_coordinator,
+      FIND_COORDINATOR_API => :find_coordinator,
       JOIN_GROUP_API => :join_group,
       HEARTBEAT_API => :heartbeat,
       LEAVE_GROUP_API => :leave_group,
@@ -70,7 +70,7 @@ module Kafka
        9 => ReplicaNotAvailable,
       10 => MessageSizeTooLarge,
       12 => OffsetMetadataTooLarge,
-      15 => GroupCoordinatorNotAvailable,
+      15 => CoordinatorNotAvailable,
       16 => NotCoordinatorForGroup,
       17 => InvalidTopic,
       18 => RecordListTooLarge,
@@ -117,6 +117,11 @@ module Kafka
       RESOURCE_TYPE_DELEGATION_TOKEN => :delegation_token,
     }
 
+    # Coordinator types. Since Kafka 0.11.0, there are types of coordinators:
+    # Group and Transaction
+    COORDINATOR_TYPE_GROUP = 0
+    COORDINATOR_TYPE_TRANSACTION = 1
+
     # Handles an error code by either doing nothing (if there was no error) or
     # by raising an appropriate exception.
     #
@@ -151,8 +156,8 @@ require "kafka/protocol/fetch_request"
 require "kafka/protocol/fetch_response"
 require "kafka/protocol/list_offset_request"
 require "kafka/protocol/list_offset_response"
-require "kafka/protocol/group_coordinator_request"
-require "kafka/protocol/group_coordinator_response"
+require "kafka/protocol/find_coordinator_request"
+require "kafka/protocol/find_coordinator_response"
 require "kafka/protocol/join_group_request"
 require "kafka/protocol/join_group_response"
 require "kafka/protocol/sync_group_request"

data/lib/kafka/protocol/find_coordinator_request.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Protocol
+    class FindCoordinatorRequest
+      def initialize(coordinator_key:, coordinator_type:)
+        @coordinator_key  = coordinator_key
+        @coordinator_type = coordinator_type
+      end
+
+      def api_key
+        FIND_COORDINATOR_API
+      end
+
+      def api_version
+        1
+      end
+
+      def encode(encoder)
+        encoder.write_string(@coordinator_key)
+        encoder.write_int8(@coordinator_type)
+      end
+
+      def response_class
+        FindCoordinatorResponse
+      end
+    end
+  end
+end

data/lib/kafka/protocol/{group_coordinator_response.rb → find_coordinator_response.rb}

@@ -2,12 +2,12 @@
 
 module Kafka
   module Protocol
-    class GroupCoordinatorResponse
-      attr_reader :error_code
+    class FindCoordinatorResponse
+      attr_reader :error_code, :error_message
 
       attr_reader :coordinator_id, :coordinator_host, :coordinator_port
 
-      def initialize(error_code:, coordinator_id:, coordinator_host:, coordinator_port:)
+      def initialize(error_code:, error_message:, coordinator_id:, coordinator_host:, coordinator_port:)
         @error_code = error_code
         @coordinator_id = coordinator_id
         @coordinator_host = coordinator_host
@@ -15,8 +15,10 @@ module Kafka
       end
 
       def self.decode(decoder)
+        _throttle_time_ms = decoder.int32
         new(
           error_code: decoder.int16,
+          error_message: decoder.string,
           coordinator_id: decoder.int32,
           coordinator_host: decoder.string,
           coordinator_port: decoder.int32,

data/lib/kafka/protocol/metadata_response.rb

@@ -33,20 +33,6 @@ module Kafka
     #         Isr => [int32]
     #
     class MetadataResponse
-      class BrokerInfo
-        attr_reader :node_id, :host, :port
-
-        def initialize(node_id:, host:, port:)
-          @node_id = node_id
-          @host = host
-          @port = port
-        end
-
-        def to_s
-          "#{host}:#{port} (node_id=#{node_id})"
-        end
-      end
-
       class PartitionMetadata
         attr_reader :partition_id, :leader
 
@@ -77,7 +63,7 @@ module Kafka
         end
       end
 
-      # @return [Array<BrokerInfo>] the list of brokers in the cluster.
+      # @return [Array<Kafka::BrokerInfo>] the list of brokers in the cluster.
       attr_reader :brokers
 
       # @return [Array<TopicMetadata>] the list of topics in the cluster.
@@ -125,7 +111,7 @@ module Kafka
       # Finds the broker info for the given node id.
       #
       # @param node_id [Integer] the node id of the broker.
-      # @return [BrokerInfo] information about the broker.
+      # @return [Kafka::BrokerInfo] information about the broker.
       def find_broker(node_id)
         broker = @brokers.find {|broker| broker.node_id == node_id }
 
@@ -134,6 +120,10 @@ module Kafka
         broker
       end
 
+      def controller_broker
+        find_broker(controller_id)
+      end
+
       def partitions_for(topic_name)
         topic = @topics.find {|t| t.topic_name == topic_name }
 

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.0.beta1"
+  VERSION = "0.7.0.beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.0.beta1
+  version: 0.7.0.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-21 00:00:00.000000000 Z
+date: 2018-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -323,6 +323,7 @@ files:
 - lib/kafka.rb
 - lib/kafka/async_producer.rb
 - lib/kafka/broker.rb
+- lib/kafka/broker_info.rb
 - lib/kafka/broker_pool.rb
 - lib/kafka/broker_uri.rb
 - lib/kafka/client.rb
@@ -370,8 +371,8 @@ files:
 - lib/kafka/protocol/encoder.rb
 - lib/kafka/protocol/fetch_request.rb
 - lib/kafka/protocol/fetch_response.rb
-- lib/kafka/protocol/group_coordinator_request.rb
-- lib/kafka/protocol/group_coordinator_response.rb
+- lib/kafka/protocol/find_coordinator_request.rb
+- lib/kafka/protocol/find_coordinator_response.rb
 - lib/kafka/protocol/heartbeat_request.rb
 - lib/kafka/protocol/heartbeat_response.rb
 - lib/kafka/protocol/join_group_request.rb

data/lib/kafka/protocol/group_coordinator_request.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Kafka
-  module Protocol
-    class GroupCoordinatorRequest
-      def initialize(group_id:)
-        @group_id = group_id
-      end
-
-      def api_key
-        GROUP_COORDINATOR_API
-      end
-
-      def encode(encoder)
-        encoder.write_string(@group_id)
-      end
-
-      def response_class
-        GroupCoordinatorResponse
-      end
-    end
-  end
-end