ruby-kafka 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c934181fd72d8f78eb9b4438b7297a6a71a51eb
-  data.tar.gz: e3a970abf6839ab03bdd34caeb53dfe9fe91666d
+  metadata.gz: 3f09ed3cd80d943c7aa595e43c3de6fd1bd0f883
+  data.tar.gz: 5906a940b05edf1ee429c184e03d4f2490b2454a
 SHA512:
-  metadata.gz: 266b2624a721b56c991797a6848531e00675274a24241db59de6d8649ac70ba41799b40d39efbfd8c90a1b467aa142fe68526533e23933c606fbaac7b9ae780d
-  data.tar.gz: 1d41e52b87201e803a3edf62a56a0086c1f8aed86dc255a9c73eee5b295acc27113e7d82adba4558666142509664610f5531c654049e596c546c89e468f153b5
+  metadata.gz: e352a9d47347f413ffafe9e89fca1369a6aa2de17da5b65de2adf3ba659784aea6939b127411626f7889941e084d57d4329b047e5aba26f83d31a5f312b3caaf
+  data.tar.gz: c1e97414fa1a314ec8e6f9f63fe992544d50c3dd6c0650ac1c2f76ba5866a0c3ed3457f38d71021f9669e6dcc904498a2f9c6ee63542234e94fd5ff000ff4c27

data/CHANGELOG.md

@@ -4,10 +4,13 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.4.4
+
+- Include the offset lag in batch consumer metrics (Statsd).
+
 ## v0.4.3
 
 - Restart the async producer thread automatically after errors.
-- Include the offset lag in batch consumer metrics (Statsd).
 - Make the default `max_wait_time` more sane.
 - Fix issue with cached default offset lookups (#431).
 - Upgrade to Datadog client version 3.

data/examples/consumer-group.rb

@@ -0,0 +1,26 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+logger = Logger.new(STDOUT)
+brokers = ENV.fetch("KAFKA_BROKERS", "localhost:9092").split(",")
+
+# Make sure to create this topic in your Kafka cluster or configure the
+# cluster to auto-create topics.
+topic = "text"
+
+kafka = Kafka.new(
+  seed_brokers: brokers,
+  client_id: "test",
+  socket_timeout: 20,
+  logger: logger,
+)
+
+consumer = kafka.consumer(group_id: "test")
+consumer.subscribe(topic)
+
+trap("TERM") { consumer.stop }
+
+consumer.each_message do |message|
+  puts message.value
+end

data/lib/kafka.rb

@@ -17,6 +17,10 @@ module Kafka
     end
   end
 
+  # A fetch operation was executed with no partitions specified.
+  class NoPartitionsToFetchFrom < Error
+  end
+
   # Subclasses of this exception class map to an error code described in the
   # Kafka protocol specification.
   #
@@ -221,9 +225,6 @@ module Kafka
   class FetchError < Error
   end
 
-  class NoPartitionsAssignedError < Error
-  end
-
   # Initializes a new Kafka client.
   #
   # @see Client#initialize

data/lib/kafka/broker.rb

@@ -10,6 +10,10 @@ module Kafka
       @logger = logger
     end
 
+    def address_match?(host, port)
+      @connection.address_match?(host, port)
+    end
+
     # @return [String]
     def to_s
       "#{@connection} (node_id=#{@node_id.inspect})"

data/lib/kafka/broker_pool.rb

@@ -9,7 +9,12 @@ module Kafka
     end
 
     def connect(host, port, node_id: nil)
-      return @brokers.fetch(node_id) if @brokers.key?(node_id)
+      if @brokers.key?(node_id)
+        broker = @brokers.fetch(node_id)
+        return broker if broker.address_match?(host, port)
+        broker.disconnect
+        @brokers[node_id] = nil
+      end
 
       broker = Broker.new(
         connection: @connection_builder.build_connection(host, port),

data/lib/kafka/cluster.rb

@@ -93,9 +93,20 @@ module Kafka
           Protocol.handle_error(response.error_code)
 
           coordinator_id = response.coordinator_id
-          coordinator = connect_to_broker(coordinator_id)
 
-          @logger.debug "Coordinator for group `#{group_id}` is #{coordinator}"
+          @logger.debug "Coordinator for group `#{group_id}` is #{coordinator_id}. Connecting..."
+
+          # It's possible that a new broker is introduced to the cluster and
+          # becomes the coordinator before we have a chance to refresh_metadata.
+          coordinator = begin
+            connect_to_broker(coordinator_id)
+          rescue Kafka::NoSuchBroker
+            @logger.debug "Broker #{coordinator_id} missing from broker cache, refreshing"
+            refresh_metadata!
+            connect_to_broker(coordinator_id)
+          end
+
+          @logger.debug "Connected to coordinator: #{coordinator} for group `#{group_id}`"
 
           return coordinator
         rescue GroupCoordinatorNotAvailable

data/lib/kafka/connection.rb

@@ -59,6 +59,10 @@ module Kafka
       @sasl_authenticator = sasl_authenticator
     end
 
+    def address_match?(host, port)
+      @host == host && @port == port
+    end
+
     def to_s
       "#{@host}:#{@port}"
     end

data/lib/kafka/consumer.rb

@@ -216,6 +216,8 @@ module Kafka
 
         # We may not have received any messages, but it's still a good idea to
         # commit offsets if we've processed messages in the last set of batches.
+        # This also ensures the offsets are retained if we haven't read any messages
+        # since the offset retention period has elapsed.
         @offset_manager.commit_offsets_if_necessary
       end
     end
@@ -279,6 +281,12 @@ module Kafka
 
           return if !@running
         end
+
+        # We may not have received any messages, but it's still a good idea to
+        # commit offsets if we've processed messages in the last set of batches.
+        # This also ensures the offsets are retained if we haven't read any messages
+        # since the offset retention period has elapsed.
+        @offset_manager.commit_offsets_if_necessary
       end
     end
 
@@ -377,8 +385,6 @@ module Kafka
 
       @heartbeat.send_if_necessary
 
-      raise NoPartitionsAssignedError if subscribed_partitions.empty?
-
       operation = FetchOperation.new(
         cluster: @cluster,
         logger: @logger,
@@ -401,6 +407,13 @@ module Kafka
       end
 
       operation.execute
+    rescue NoPartitionsToFetchFrom
+      backoff = max_wait_time > 0 ? max_wait_time : 1
+
+      @logger.info "There are no partitions to fetch from, sleeping for #{backoff}s"
+      sleep backoff
+
+      retry
     rescue OffsetOutOfRange => e
       @logger.error "Invalid offset for #{e.topic}/#{e.partition}, resetting to default offset"
 

data/lib/kafka/fetch_operation.rb

@@ -46,6 +46,10 @@ module Kafka
 
       topics_by_broker = {}
 
+      if @topics.none? {|topic, partitions| partitions.any? }
+        raise NoPartitionsToFetchFrom
+      end
+
       @topics.each do |topic, partitions|
         partitions.each do |partition, options|
           broker = @cluster.get_leader(topic, partition)

data/lib/kafka/statsd.rb

@@ -97,6 +97,7 @@ module Kafka
       end
 
       def process_batch(event)
+        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
@@ -109,6 +110,8 @@ module Kafka
           timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
           count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
         end
+
+        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
       end
 
       def join_group(event)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.4.3"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-20 00:00:00.000000000 Z
+date: 2017-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -272,6 +272,7 @@ files:
 - ci/init.rb
 - ci/producer.rb
 - circle.yml
+- examples/consumer-group.rb
 - examples/firehose-consumer.rb
 - examples/firehose-producer.rb
 - examples/simple-consumer.rb
@@ -380,7 +381,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: A client library for the Kafka distributed commit log.