ruby-kafka 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf114586f8ec65d1db3ade1d082fc82de140f241
-  data.tar.gz: a202f5dd06b0339a8e3895e48824acba6d266ba5
+  metadata.gz: faf979f6512df3c644b20c1fad027fdc2818d558
+  data.tar.gz: 729c7d34332b6c832fdcc1bd6ee98daaf8af09e6
 SHA512:
-  metadata.gz: 4788eabf90baf70adb425860c98073058dd601247f1dd447aeb3d154025a5466c19130c7a8d00984ff0e9a5771f980a39a33ffec8bc4384695388999ceda2a13
-  data.tar.gz: 8f5572d8e9dcf46737236ac6a6467fc5d1bd60e091e889ffe056131c084bd47699c59bd404e11b0ce8ec3c5b5c800d8d07c318c6d3abd2e75585d4168079c70c
+  metadata.gz: c3ef737a7b4af9f94413759c4d7c1dec136f727f3bf7963cc3b7daaf26d42600046deca40cd28c2ba8685b7d54e4efbb841194fe272218b24a70abf76b359e51
+  data.tar.gz: 691a8b02e96485959653c84fc5fa83d586a569e2866a94a30a490de9031a37beea9c96d71e8e244a9bfde0a779f6066cea3cd3d53253142da6d2c95a702b38d2

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.4
+
+- Make the producer buffer more resilient in the face of isolated topic errors.
+
 ## v0.3.3
 
 - Allow clearing a producer's buffer (Martin Nowak).

data/examples/firehose-consumer.rb

@@ -9,7 +9,7 @@ KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")
 KAFKA_SERVER_CERT = ENV.fetch("KAFKA_SERVER_CERT")
 KAFKA_URL = ENV.fetch("KAFKA_URL")
-KAFKA_BROKERS = KAFKA_URL.gsub("kafka+ssl://", "").split(",")
+KAFKA_BROKERS = KAFKA_URL
 KAFKA_TOPIC = "test-messages"
 
 NUM_THREADS = 4

data/examples/firehose-producer.rb

@@ -9,7 +9,7 @@ KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")
 KAFKA_SERVER_CERT = ENV.fetch("KAFKA_SERVER_CERT")
 KAFKA_URL = ENV.fetch("KAFKA_URL")
-KAFKA_BROKERS = KAFKA_URL.gsub("kafka+ssl://", "").split(",")
+KAFKA_BROKERS = KAFKA_URL
 KAFKA_TOPIC = "test-messages"
 
 NUM_THREADS = 20

data/examples/simple-consumer.rb

@@ -14,7 +14,7 @@ require "kafka"
 # with e.g. `$stderr` if you want to see what's happening under the hood.
 logger = Logger.new(StringIO.new)
 
-brokers = ENV.fetch("KAFKA_BROKERS").split(",")
+brokers = ENV.fetch("KAFKA_BROKERS")
 
 # Make sure to create this topic in your Kafka cluster or configure the
 # cluster to auto-create topics.

data/examples/simple-producer.rb

@@ -11,7 +11,7 @@ $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
 require "kafka"
 
 logger = Logger.new($stderr)
-brokers = ENV.fetch("KAFKA_BROKERS").split(",")
+brokers = ENV.fetch("KAFKA_BROKERS")
 
 # Make sure to create this topic in your Kafka cluster or configure the
 # cluster to auto-create topics.

data/examples/ssl-producer.rb

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
 require "kafka"
 
 logger = Logger.new($stderr)
-brokers = ENV.fetch("KAFKA_BROKERS").split(",")
+brokers = ENV.fetch("KAFKA_BROKERS")
 
 # Make sure to create this topic in your Kafka cluster or configure the
 # cluster to auto-create topics.

data/lib/kafka/client.rb

@@ -15,8 +15,10 @@ module Kafka
 
     # Initializes a new Kafka client.
     #
-    # @param seed_brokers [Array<String>] the list of brokers used to initialize
-    #   the client.
+    # @param seed_brokers [Array<String>, String] the list of brokers used to initialize
+    #   the client. Either an Array of connections, or a comma separated string of connections.
+    #   Connections can either be a string of "port:protocol" or a full URI with a scheme.
+    #   If there's a scheme it's ignored and only host/port are used.
     #
     # @param client_id [String] the identifier for this application.
     #
@@ -59,7 +61,7 @@ module Kafka
       )
 
       @cluster = Cluster.new(
-        seed_brokers: seed_brokers,
+        seed_brokers: normalize_seed_brokers(seed_brokers),
         broker_pool: broker_pool,
         logger: @logger,
       )
@@ -307,5 +309,21 @@ module Kafka
 
       ssl_context
     end
+
+    def normalize_seed_brokers(seed_brokers)
+      if seed_brokers.is_a?(String)
+        seed_brokers = seed_brokers.split(",")
+      end
+      brokers = []
+      seed_brokers.each do |connection|
+        if connection =~ /:\/\//
+          u = URI.parse(connection)
+          brokers << "#{u.host}:#{u.port}"
+        else
+          brokers << connection
+        end
+      end
+      brokers
+    end
   end
 end

data/lib/kafka/pending_message_queue.rb

@@ -1,8 +1,5 @@
 module Kafka
 
-  # A pending message queue holds messages that have not yet been assigned to
-  # a partition. It's designed to only remove messages once they've been
-  # successfully handled.
   class PendingMessageQueue
     attr_reader :size, :bytesize
 
@@ -26,22 +23,17 @@ module Kafka
       @bytesize = 0
     end
 
-    # Yields each message in the queue to the provided block, removing the
-    # message after the block has processed it. If the block raises an
-    # exception, the message will be retained in the queue.
+    def replace(messages)
+      clear
+      messages.each {|message| write(message) }
+    end
+
+    # Yields each message in the queue.
     #
     # @yieldparam [PendingMessage] message
     # @return [nil]
-    def dequeue_each(&block)
-      until @messages.empty?
-        message = @messages.first
-
-        yield message
-
-        @size -= 1
-        @bytesize -= message.bytesize
-        @messages.shift
-      end
+    def each(&block)
+      @messages.each(&block)
     end
   end
 end

data/lib/kafka/produce_operation.rb

@@ -65,9 +65,8 @@ module Kafka
         rescue Kafka::Error => e
           @logger.error "Could not connect to leader for partition #{topic}/#{partition}: #{e.message}"
 
-          @instrumenter.instrument("partition_error.producer", {
+          @instrumenter.instrument("topic_error.producer", {
             topic: topic,
-            partition: partition,
             exception: [e.class.to_s, e.message],
           })
 
@@ -118,9 +117,8 @@ module Kafka
           begin
             Protocol.handle_error(partition_info.error_code)
           rescue ProtocolError => e
-            @instrumenter.instrument("partition_error.producer", {
+            @instrumenter.instrument("topic_error.producer", {
               topic: topic,
-              partition: partition,
               exception: [e.class.to_s, e.message],
             })
 

data/lib/kafka/producer.rb

@@ -327,25 +327,40 @@ module Kafka
     end
 
     def assign_partitions!
-      @pending_message_queue.dequeue_each do |message|
+      failed_messages = []
+
+      @pending_message_queue.each do |message|
         partition = message.partition
 
-        if partition.nil?
-          partition_count = @cluster.partitions_for(message.topic).count
-          partition = Partitioner.partition_for_key(partition_count, message)
+        begin
+          if partition.nil?
+            partition_count = @cluster.partitions_for(message.topic).count
+            partition = Partitioner.partition_for_key(partition_count, message)
+          end
+
+          @buffer.write(
+            value: message.value,
+            key: message.key,
+            topic: message.topic,
+            partition: partition,
+            create_time: message.create_time,
+          )
+        rescue Kafka::Error => e
+          @instrumenter.instrument("topic_error.producer", {
+            topic: message.topic,
+            exception: [e.class.to_s, e.message],
+          })
+
+          failed_messages << message
         end
+      end
 
-        @buffer.write(
-          value: message.value,
-          key: message.key,
-          topic: message.topic,
-          partition: partition,
-          create_time: message.create_time,
-        )
+      if failed_messages.any?
+        @logger.error "Failed to assign partitions to #{failed_messages.count} messages"
+        @cluster.mark_as_stale!
       end
-    rescue Kafka::Error => e
-      @logger.error "Failed to assign pending message to a partition: #{e}"
-      @cluster.mark_as_stale!
+
+      @pending_message_queue.replace(failed_messages)
     end
   end
 end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Daniel Schierbeck