ruby-kafka 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87132b87fc32443be48994590b059a88b6cc5fea
-  data.tar.gz: c32e7ed87e9dcc7c2ba6f3128db0aadb123d1962
+  metadata.gz: 2842846dbe233b635e3dae0c3e14d9d660d19089
+  data.tar.gz: f7a9b7a9aea3df0e3175b0b239fe9d72f743fed5
 SHA512:
-  metadata.gz: 84d5def3fa8963f928d2e59239e9c2678cfbe2ba904ad67bc13080a16b861bb5296394c442c14853144d065a4b86f205a8ba378e28302f67f8b80c5eaf0b7100
-  data.tar.gz: b39147eb0fd72f1753af38831871a00f597d0191a70f3f47734f87a2212b7d1500c0b1e712c36755b7ab09b27d66a26ea020f6a2fc413ea9ed9a0e2e15449341
+  metadata.gz: a4c354e1677e76d0d0eb5483fd56b9d0fba7b8d16baf0f1fe450960977e8c18edba2c00d0a650168d280d03784170e078ae77e7914b6451bd20fc1784aea4867
+  data.tar.gz: 25e871eb218789b094a167371560db89edb1be38387969c2c74fa892701c527abb9d1cd9be0b14b41b65e4805cbbab19dd729c56c963b7555f6d7e15136ea20d

data/CHANGELOG.md

@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.1
+
+- Simplify the heartbeat algorithm.
+- Handle partial messages at the end of message sets received from the brokers.
+
 ## v0.3.0
 
 - Add support for encryption and authentication with SSL (Tom Crayford).

data/README.md

@@ -18,11 +18,11 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
     5. [Message Delivery Guarantees](#message-delivery-guarantees)
     6. [Compression](#compression)
   2. [Consuming Messages from Kafka](#consuming-messages-from-kafka)
-3. [Logging](#logging)
-4. [Understanding Timeouts](#understanding-timeouts)
-5. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
-6. [Development](#development)
-7. [Roadmap](#roadmap)
+  3. [Logging](#logging)
+  4. [Understanding Timeouts](#understanding-timeouts)
+  5. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
+3. [Development](#development)
+4. [Roadmap](#roadmap)
 
 ## Installation
 

data/examples/firehose-consumer.rb

@@ -0,0 +1,63 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
+KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")
+KAFKA_SERVER_CERT = ENV.fetch("KAFKA_SERVER_CERT")
+KAFKA_URL = ENV.fetch("KAFKA_URL")
+KAFKA_BROKERS = KAFKA_URL.gsub("kafka+ssl://", "").split(",")
+KAFKA_TOPIC = "test-messages"
+
+NUM_THREADS = 4
+
+queue = Queue.new
+
+threads = NUM_THREADS.times.map do |worker_id|
+  Thread.new do
+    logger = Logger.new($stderr)
+    logger.level = Logger::INFO
+
+    logger.formatter = proc {|severity, datetime, progname, msg|
+      "[#{worker_id}] #{severity.ljust(5)} -- #{msg}\n"
+    }
+
+    kafka = Kafka.new(
+      seed_brokers: KAFKA_BROKERS,
+      logger: logger,
+      connect_timeout: 30,
+      socket_timeout: 30,
+      ssl_client_cert: KAFKA_CLIENT_CERT,
+      ssl_client_cert_key: KAFKA_CLIENT_CERT_KEY,
+      ssl_ca_cert: KAFKA_SERVER_CERT,
+    )
+
+    consumer = kafka.consumer(group_id: "firehose")
+    consumer.subscribe(KAFKA_TOPIC)
+
+    begin
+      i = 0
+      consumer.each_message do |message|
+        i += 1
+
+        if i % 1000 == 0
+          queue << i
+          i = 0
+        end
+
+        sleep 0.01
+      end
+    ensure
+      consumer.shutdown
+    end
+  end
+end
+
+threads.each {|t| t.abort_on_exception = true }
+
+received_messages = 0
+
+loop do
+  received_messages += queue.pop
+  puts "===> Received #{received_messages} messages"
+end

data/examples/firehose-producer.rb

@@ -1,6 +1,9 @@
 $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
 
 require "kafka"
+require "dotenv"
+
+Dotenv.load
 
 KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")

data/lib/kafka/consumer.rb

@@ -57,6 +57,9 @@ module Kafka
       @group = group
       @offset_manager = offset_manager
       @session_timeout = session_timeout
+
+      # Send two heartbeats in each session window, just to be sure.
+      @heartbeat_interval = @session_timeout / 2
     end
 
     # Subscribes the consumer to a topic.
@@ -122,6 +125,9 @@ module Kafka
         rescue RebalanceInProgress
           @logger.error "Group is rebalancing; rejoining"
           join_group
+        rescue IllegalGeneration
+          @logger.error "Group has transitioned to a new generation; rejoining"
+          join_group
         end
       end
     end
@@ -197,9 +203,9 @@ module Kafka
     # `session_timeout`.
     #
     def send_heartbeat_if_necessary
-      @last_heartbeat ||= Time.at(0)
+      @last_heartbeat ||= Time.now
 
-      if @last_heartbeat <= Time.now - @session_timeout + 2
+      if Time.now > @last_heartbeat + @heartbeat_interval
         @group.heartbeat
         @last_heartbeat = Time.now
       end

data/lib/kafka/consumer_group.rb

@@ -28,6 +28,10 @@ module Kafka
     end
 
     def join
+      if @topics.empty?
+        raise Kafka::Error, "Cannot join group without at least one topic subscription"
+      end
+
       join_group
       synchronize
     rescue NotCoordinatorForGroup
@@ -42,7 +46,7 @@ module Kafka
     end
 
     def leave
-      @logger.info "[#{@member_id}] Leaving group `#{@group_id}`"
+      @logger.info "Leaving group `#{@group_id}`"
       coordinator.leave_group(group_id: @group_id, member_id: @member_id)
     rescue ConnectionError
     end
@@ -70,7 +74,7 @@ module Kafka
     end
 
     def heartbeat
-      @logger.info "[#{@member_id}] Sending heartbeat..."
+      @logger.info "Sending heartbeat..."
 
       response = coordinator.heartbeat(
         group_id: @group_id,
@@ -99,7 +103,7 @@ module Kafka
       @leader_id = response.leader_id
       @members = response.members
 
-      @logger.info "[#{@member_id}] Joined group `#{@group_id}` with member id `#{@member_id}`"
+      @logger.info "Joined group `#{@group_id}` with member id `#{@member_id}`"
     rescue UnknownMemberId
       @logger.error "Failed to join group; resetting member id and retrying in 1s..."
 
@@ -117,7 +121,7 @@ module Kafka
       group_assignment = {}
 
       if group_leader?
-        @logger.info "[#{@member_id}] Chosen as leader of group `#{@group_id}`"
+        @logger.info "Chosen as leader of group `#{@group_id}`"
 
         group_assignment = @assignment_strategy.assign(
           members: @members.keys,
@@ -135,7 +139,7 @@ module Kafka
       Protocol.handle_error(response.error_code)
 
       response.member_assignment.topics.each do |topic, assigned_partitions|
-        @logger.info "[#{@member_id}] Partitions assigned for `#{topic}`: #{assigned_partitions.join(', ')}"
+        @logger.info "Partitions assigned for `#{topic}`: #{assigned_partitions.join(', ')}"
       end
 
       @assigned_partitions.replace(response.member_assignment.topics)

data/lib/kafka/protocol/message_set.rb

@@ -27,13 +27,17 @@ module Kafka
         fetched_messages = []
 
         until decoder.eof?
-          message = Message.decode(decoder)
-
-          if message.compressed?
-            wrapped_message_set = message.decompress
-            fetched_messages.concat(wrapped_message_set.messages)
-          else
-            fetched_messages << message
+          begin
+            message = Message.decode(decoder)
+
+            if message.compressed?
+              wrapped_message_set = message.decompress
+              fetched_messages.concat(wrapped_message_set.messages)
+            else
+              fetched_messages << message
+            end
+          rescue EOFError
+            # We tried to decode a partial message; just skip it.
           end
         end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -163,6 +163,7 @@ files:
 - bin/console
 - bin/setup
 - circle.yml
+- examples/firehose-consumer.rb
 - examples/firehose-producer.rb
 - examples/simple-consumer.rb
 - examples/simple-producer.rb