ruby-kafka 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9845964409d385d935b55e4ded95f210721fe7ce
-  data.tar.gz: d7bebf3d0222a80c0fe95ebec4d83b992abe306d
+  metadata.gz: 5c001c97f73b001e170f23d7fd970781aa6bb196
+  data.tar.gz: 39f0b03b56ad97eee788fac63ec467599ad8cd8e
 SHA512:
-  metadata.gz: 5656f9723720414c954a7087385d52d9618322d31e70f9600bd4b8f34bb76432a31fb0bc88bc6baf5506fe518c47a7a5c9cb0134c91f224acdaa6ef76bbd8fce
-  data.tar.gz: 042d3a1bf5f8847e9b50a93e2bc8b151e78c4fac463e999e32e9d0acd7f1d5498ebb301d166a8a255adcc2efd15f7bd53de6397348a94822f2462c626c49741f
+  metadata.gz: 6f9312f25ab7493d5803b581b22af270399f0f5ff2fbf4012da1f4a30525c430fb1fa21085cb4042aa4b59074a0abe3af9038ee6e97f0b48770baa50eaab6393
+  data.tar.gz: 7523013dbc08d7a193faf6b2b2419a6704cb4b7918f9ed993545b534e356084a765a1c8cfba937a55d7236c26985e3cf252cbe50df6c3c98a45e1bc47e32caad

data/CHANGELOG.md

@@ -4,6 +4,8 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.4.1
+
 - Allow seeking the consumer position (#386).
 - Reopen idle connections after 5 minutes (#399).
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ruby-kafka (0.4.0)
+    ruby-kafka (0.4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -89,4 +89,4 @@ RUBY VERSION
    ruby 2.2.3p173
 
 BUNDLED WITH
-   1.14.6
+   1.15.3

data/lib/kafka/async_producer.rb

@@ -83,6 +83,8 @@ module Kafka
         queue: @queue,
         producer: sync_producer,
         delivery_threshold: delivery_threshold,
+        instrumenter: instrumenter,
+        logger: logger,
       )
 
       # The timer will no-op if the delivery interval is zero.
@@ -180,10 +182,12 @@ module Kafka
     end
 
     class Worker
-      def initialize(queue:, producer:, delivery_threshold:)
+      def initialize(queue:, producer:, delivery_threshold:, instrumenter:, logger:)
         @queue = queue
         @producer = producer
         @delivery_threshold = delivery_threshold
+        @instrumenter = instrumenter
+        @logger = logger
       end
 
       def run
@@ -197,8 +201,16 @@ module Kafka
           when :deliver_messages
             deliver_messages
           when :shutdown
-            # Deliver any pending messages first.
-            deliver_messages
+            begin
+              # Deliver any pending messages first.
+              @producer.deliver_messages
+            rescue Error => e
+              @logger.error("Failed to deliver messages during shutdown: #{e.message}")
+
+              @instrumenter.instrument("drop_messages.async_producer", {
+                message_count: @producer.buffer_size + @queue.size,
+              })
+            end
 
             # Stop the run loop.
             break

data/lib/kafka/client.rb

@@ -10,6 +10,7 @@ require "kafka/fetched_message"
 require "kafka/fetch_operation"
 require "kafka/connection_builder"
 require "kafka/instrumenter"
+require "kafka/sasl_authenticator"
 
 module Kafka
   class Client
@@ -58,6 +59,15 @@ module Kafka
 
       ssl_context = build_ssl_context(ssl_ca_cert_file_path, ssl_ca_cert, ssl_client_cert, ssl_client_cert_key)
 
+      sasl_authenticator = SaslAuthenticator.new(
+        sasl_gssapi_principal: sasl_gssapi_principal,
+        sasl_gssapi_keytab: sasl_gssapi_keytab,
+        sasl_plain_authzid: sasl_plain_authzid,
+        sasl_plain_username: sasl_plain_username,
+        sasl_plain_password: sasl_plain_password,
+        logger: @logger
+      )
+
       @connection_builder = ConnectionBuilder.new(
         client_id: client_id,
         connect_timeout: connect_timeout,
@@ -65,11 +75,7 @@ module Kafka
         ssl_context: ssl_context,
         logger: @logger,
         instrumenter: @instrumenter,
-        sasl_gssapi_principal: sasl_gssapi_principal,
-        sasl_gssapi_keytab: sasl_gssapi_keytab,
-        sasl_plain_authzid: sasl_plain_authzid,
-        sasl_plain_username: sasl_plain_username,
-        sasl_plain_password: sasl_plain_password
+        sasl_authenticator: sasl_authenticator
       )
 
       @cluster = initialize_cluster
@@ -248,7 +254,8 @@ module Kafka
         logger: @logger,
         group_id: group_id,
         session_timeout: session_timeout,
-        retention_time: retention_time
+        retention_time: retention_time,
+        instrumenter: instrumenter,
       )
 
       offset_manager = OffsetManager.new(

data/lib/kafka/cluster.rb

@@ -193,6 +193,8 @@ module Kafka
     # @raise [ConnectionError] if none of the nodes in `seed_brokers` are available.
     # @return [Protocol::MetadataResponse] the cluster metadata.
     def fetch_cluster_info
+      errors = []
+
       @seed_brokers.shuffle.each do |node|
         @logger.info "Fetching cluster metadata from #{node}"
 
@@ -207,12 +209,15 @@ module Kafka
           return cluster_info
         rescue Error => e
           @logger.error "Failed to fetch metadata from #{node}: #{e}"
+          errors << [node, e]
         ensure
           broker.disconnect unless broker.nil?
         end
       end
 
-      raise ConnectionError, "Could not connect to any of the seed brokers: #{@seed_brokers.join(', ')}"
+      error_description = errors.map {|node, exception| "- #{node}: #{exception}" }.join("\n")
+
+      raise ConnectionError, "Could not connect to any of the seed brokers:\n#{error_description}"
     end
 
     def connect_to_broker(broker_id)

data/lib/kafka/connection.rb

@@ -48,7 +48,7 @@ module Kafka
     #   broker. Default is 10 seconds.
     #
     # @return [Connection] a new connection.
-    def initialize(host:, port:, client_id:, logger:, instrumenter:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
+    def initialize(host:, port:, client_id:, logger:, instrumenter:, sasl_authenticator:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
       @host, @port, @client_id = host, port, client_id
       @logger = logger
       @instrumenter = instrumenter
@@ -56,6 +56,7 @@ module Kafka
       @connect_timeout = connect_timeout || CONNECT_TIMEOUT
       @socket_timeout = socket_timeout || SOCKET_TIMEOUT
       @ssl_context = ssl_context
+      @sasl_authenticator = sasl_authenticator
     end
 
     def to_s
@@ -128,6 +129,7 @@ module Kafka
       @correlation_id = 0
 
       @last_request = nil
+      @sasl_authenticator.authenticate!(self)
     rescue Errno::ETIMEDOUT => e
       @logger.error "Timed out while trying to connect to #{self}: #{e}"
       raise ConnectionError, e

data/lib/kafka/connection_builder.rb

@@ -1,20 +1,13 @@
-require 'kafka/sasl_gssapi_authenticator'
-require 'kafka/sasl_plain_authenticator'
-
 module Kafka
   class ConnectionBuilder
-    def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_gssapi_principal:, sasl_gssapi_keytab:, sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:)
+    def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_authenticator:)
       @client_id = client_id
       @logger = logger
       @instrumenter = instrumenter
       @connect_timeout = connect_timeout
       @socket_timeout = socket_timeout
       @ssl_context = ssl_context
-      @sasl_gssapi_principal = sasl_gssapi_principal
-      @sasl_gssapi_keytab = sasl_gssapi_keytab
-      @sasl_plain_authzid = sasl_plain_authzid
-      @sasl_plain_username = sasl_plain_username
-      @sasl_plain_password = sasl_plain_password
+      @sasl_authenticator = sasl_authenticator
     end
 
     def build_connection(host, port)
@@ -26,49 +19,12 @@ module Kafka
         socket_timeout: @socket_timeout,
         logger: @logger,
         instrumenter: @instrumenter,
-        ssl_context: @ssl_context
+        ssl_context: @ssl_context,
+        sasl_authenticator: @sasl_authenticator
       )
 
-      if authenticate_using_sasl_gssapi?
-        sasl_gssapi_authenticate(connection)
-      elsif authenticate_using_sasl_plain?
-        sasl_plain_authenticate(connection)
-      end
-
       connection
     end
 
-    private
-
-    def sasl_gssapi_authenticate(connection)
-      auth = SaslGssapiAuthenticator.new(
-        connection: connection,
-        logger: @logger,
-        sasl_gssapi_principal: @sasl_gssapi_principal,
-        sasl_gssapi_keytab: @sasl_gssapi_keytab
-      )
-
-      auth.authenticate!
-    end
-
-    def sasl_plain_authenticate(connection)
-      auth = SaslPlainAuthenticator.new(
-        connection: connection,
-        logger: @logger,
-        authzid: @sasl_plain_authzid,
-        username: @sasl_plain_username,
-        password: @sasl_plain_password
-      )
-
-      auth.authenticate!
-    end
-
-    def authenticate_using_sasl_gssapi?
-      !@ssl_context && @sasl_gssapi_principal && !@sasl_gssapi_principal.empty?
-    end
-
-    def authenticate_using_sasl_plain?
-      @sasl_plain_authzid && @sasl_plain_username && @sasl_plain_password
-    end
   end
 end

data/lib/kafka/consumer_group.rb

@@ -5,11 +5,12 @@ module Kafka
   class ConsumerGroup
     attr_reader :assigned_partitions, :generation_id
 
-    def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:)
+    def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:, instrumenter:)
       @cluster = cluster
       @logger = logger
       @group_id = group_id
       @session_timeout = session_timeout
+      @instrumenter = instrumenter
       @member_id = ""
       @generation_id = nil
       @members = {}
@@ -53,7 +54,10 @@ module Kafka
 
     def leave
       @logger.info "Leaving group `#{@group_id}`"
-      coordinator.leave_group(group_id: @group_id, member_id: @member_id)
+
+      @instrumenter.instrument("leave_group.consumer", group_id: @group_id) do
+        coordinator.leave_group(group_id: @group_id, member_id: @member_id)
+      end
     rescue ConnectionError
     end
 
@@ -108,18 +112,20 @@ module Kafka
     def join_group
       @logger.info "Joining group `#{@group_id}`"
 
-      response = coordinator.join_group(
-        group_id: @group_id,
-        session_timeout: @session_timeout,
-        member_id: @member_id,
-      )
+      @instrumenter.instrument("join_group.consumer", group_id: @group_id) do
+        response = coordinator.join_group(
+          group_id: @group_id,
+          session_timeout: @session_timeout,
+          member_id: @member_id,
+        )
 
-      Protocol.handle_error(response.error_code)
+        Protocol.handle_error(response.error_code)
 
-      @generation_id = response.generation_id
-      @member_id = response.member_id
-      @leader_id = response.leader_id
-      @members = response.members
+        @generation_id = response.generation_id
+        @member_id = response.member_id
+        @leader_id = response.leader_id
+        @members = response.members
+      end
 
       @logger.info "Joined group `#{@group_id}` with member id `#{@member_id}`"
     rescue UnknownMemberId
@@ -147,20 +153,22 @@ module Kafka
         )
       end
 
-      response = coordinator.sync_group(
-        group_id: @group_id,
-        generation_id: @generation_id,
-        member_id: @member_id,
-        group_assignment: group_assignment,
-      )
+      @instrumenter.instrument("sync_group.consumer", group_id: @group_id) do
+        response = coordinator.sync_group(
+          group_id: @group_id,
+          generation_id: @generation_id,
+          member_id: @member_id,
+          group_assignment: group_assignment,
+        )
 
-      Protocol.handle_error(response.error_code)
+        Protocol.handle_error(response.error_code)
 
-      response.member_assignment.topics.each do |topic, assigned_partitions|
-        @logger.info "Partitions assigned for `#{topic}`: #{assigned_partitions.join(', ')}"
-      end
+        response.member_assignment.topics.each do |topic, assigned_partitions|
+          @logger.info "Partitions assigned for `#{topic}`: #{assigned_partitions.join(', ')}"
+        end
 
-      @assigned_partitions.replace(response.member_assignment.topics)
+        @assigned_partitions.replace(response.member_assignment.topics)
+      end
     end
 
     def coordinator

data/lib/kafka/datadog.rb

@@ -130,6 +130,45 @@ module Kafka
         end
       end
 
+      def join_group(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        timing("consumer.join_group", event.duration, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("consumer.join_group.errors", tags: tags)
+        end
+      end
+
+      def sync_group(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        timing("consumer.sync_group", event.duration, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("consumer.sync_group.errors", tags: tags)
+        end
+      end
+
+      def leave_group(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        timing("consumer.leave_group", event.duration, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("consumer.leave_group.errors", tags: tags)
+        end
+      end
+
       attach_to "consumer.kafka"
     end
 
@@ -145,6 +184,7 @@ module Kafka
 
         tags = {
           client: client,
+          topic: topic,
         }
 
         # This gets us the write rate.
@@ -226,6 +266,7 @@ module Kafka
 
         tags = {
           client: client,
+          topic: topic,
         }
 
         # This gets us the avg/max queue size per producer.
@@ -244,6 +285,16 @@ module Kafka
         increment("async_producer.produce.errors", tags: tags)
       end
 
+      def drop_messages(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+        }
+
+        message_count = event.payload.fetch(:message_count)
+
+        count("async_producer.dropped_messages", message_count, tags: tags)
+      end
+
       attach_to "async_producer.kafka"
     end
   end

data/lib/kafka/offset_manager.rb

@@ -30,9 +30,9 @@ module Kafka
       @processed_offsets[topic] ||= {}
 
       # The committed offset should always be the offset of the next message that the
-      # application will read, thus adding one to the last message processed
+      # application will read, thus adding one to the last message processed.
       @processed_offsets[topic][partition] = offset + 1
-      @logger.debug "Marking #{topic}/#{partition}:#{offset} as committed"
+      @logger.debug "Marking #{topic}/#{partition}:#{offset} as processed"
     end
 
     def seek_to_default(topic, partition)

data/lib/kafka/sasl_authenticator.rb

@@ -0,0 +1,56 @@
+require 'kafka/sasl_gssapi_authenticator'
+require 'kafka/sasl_plain_authenticator'
+
+module Kafka
+  class SaslAuthenticator
+    def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:, sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:)
+      @logger = logger
+      @sasl_gssapi_principal = sasl_gssapi_principal
+      @sasl_gssapi_keytab = sasl_gssapi_keytab
+      @sasl_plain_authzid = sasl_plain_authzid
+      @sasl_plain_username = sasl_plain_username
+      @sasl_plain_password = sasl_plain_password
+    end
+
+    def authenticate!(connection)
+      if authenticate_using_sasl_gssapi?
+        sasl_gssapi_authenticate(connection)
+      elsif authenticate_using_sasl_plain?
+        sasl_plain_authenticate(connection)
+      end
+    end
+
+    private
+
+    def sasl_gssapi_authenticate(connection)
+      auth = SaslGssapiAuthenticator.new(
+        connection: connection,
+        logger: @logger,
+        sasl_gssapi_principal: @sasl_gssapi_principal,
+        sasl_gssapi_keytab: @sasl_gssapi_keytab
+      )
+
+      auth.authenticate!
+    end
+
+    def sasl_plain_authenticate(connection)
+      auth = SaslPlainAuthenticator.new(
+        connection: connection,
+        logger: @logger,
+        authzid: @sasl_plain_authzid,
+        username: @sasl_plain_username,
+        password: @sasl_plain_password
+      )
+
+      auth.authenticate!
+    end
+
+    def authenticate_using_sasl_gssapi?
+      !@ssl_context && @sasl_gssapi_principal && !@sasl_gssapi_principal.empty?
+    end
+
+    def authenticate_using_sasl_plain?
+      @sasl_plain_authzid && @sasl_plain_username && @sasl_plain_password
+    end
+  end
+end

data/lib/kafka/statsd.rb

@@ -111,6 +111,39 @@ module Kafka
         end
       end
 
+      def join_group(event)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        timing("consumer.#{client}.#{group_id}.join_group", event.duration)
+
+        if event.payload.key?(:exception)
+          increment("consumer.#{client}.#{group_id}.join_group.errors")
+        end
+      end
+
+      def sync_group(event)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        timing("consumer.#{client}.#{group_id}.sync_group", event.duration)
+
+        if event.payload.key?(:exception)
+          increment("consumer.#{client}.#{group_id}.sync_group.errors")
+        end
+      end
+
+      def leave_group(event)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        timing("consumer.#{client}.#{group_id}.leave_group", event.duration)
+
+        if event.payload.key?(:exception)
+          increment("consumer.#{client}.#{group_id}.leave_group.errors")
+        end
+      end
+
       attach_to "consumer.kafka"
     end
 
@@ -205,6 +238,13 @@ module Kafka
         increment("async_producer.#{client}.#{topic}.produce.errors")
       end
 
+      def drop_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:message_count)
+
+        count("async_producer.#{client}.dropped_messages", message_count)
+      end
+
       attach_to "async_producer.kafka"
     end
   end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -322,6 +322,7 @@ files:
 - lib/kafka/protocol/sync_group_response.rb
 - lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/round_robin_assignment_strategy.rb
+- lib/kafka/sasl_authenticator.rb
 - lib/kafka/sasl_gssapi_authenticator.rb
 - lib/kafka/sasl_plain_authenticator.rb
 - lib/kafka/snappy_codec.rb