ruby-kafka 0.7.1.beta2 → 0.7.1.debugcorruption2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: def137ba9f120b344452139b52dd8826740af0343fd17da25f96ee819e1f7c87
-  data.tar.gz: 8ffb491a63f5a7fbea9771962e521169d11ea84c810effecbf0140ebf1a8f1fe
+  metadata.gz: 309f92b03ff0cd0687e06bb794ca1881d33abec51cd388145aa45bd8adfe1f89
+  data.tar.gz: 990eeb531683ceaba2ef825a97847a71b9cc23e59c25bfb817ca98098d4d2a6b
 SHA512:
-  metadata.gz: 21e457a2830b7e3524654b4e04fd939a4532fc4f531eab0c0bb2572e71a5426fd8ec484721c8ae40b740c77f3be1762642ec698bb1e6177f502d20952509320a
-  data.tar.gz: cda0dcf607d826c027eb4c91f923038353048b22296a79a4cfa2e9edc16adeb83e50ed66d992aba4335c9ee94dcba0bba06c5b0c7f7ece3b6595e90ebbbba6d5
+  metadata.gz: 35336add7284db579083c04e67fed4a4ecc0274bad6ef015f55c77f84e5feb0846f3dfb3729aeb9a1345cad22ee152b8c6d6e8f77474c497c4300e0441ab3712
+  data.tar.gz: b4b6ccfdaa9decaabb5225b8b0118a3b9e1c7a2e670748479426e47757413f0f59f2fcbddd2db89f7e7f7189421592536f8b7e2b850154a74aa68c8ef8f572db

data/CHANGELOG.md

@@ -4,11 +4,11 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
-- Fix consuming duplication due to redundant messages returned from Kafka (#636).
-- Fresh cluster info on fetch error (#641).
-
 ## 0.7.1
 
+- Compatibility with dogstatsd-ruby v4.0.0.
+- Fix consuming duplication due to redundant messages returned from Kafka (#636).
+- Fresh cluster info on fetch error (#641).
 - Exactly Once Delivery and Transactional Messaging Support (#608).
 - Support extra client certificates in the SSL Context when authenticating with Kafka (#633).
 

data/lib/kafka/broker.rb

@@ -71,7 +71,31 @@ module Kafka
     def produce(**options)
       request = Protocol::ProduceRequest.new(**options)
 
-      send_request(request)
+      resp = send_request(request)
+
+      return if resp.nil?
+
+      errored_partitions = []
+      resp.topics.each do |topic_info|
+        errored = topic_info.partitions.find {|part_info| part_info.error_code == 2 }
+        errored_partitions << errored unless errored.nil?
+      end
+
+      errored_partitions.each do |partition_info|
+
+        @logger.error "Corrupt message code received for #{request} to partition #{partition_info.partition}"
+
+        #{"topic" => {"partition" => [message, message] }}
+        request.messages_for_topics.each do |topic, messages_for_partition|
+          messages_for_partition.fetch(partition_info.partition, []).each do |message|
+            @logger.error "Message key: #{message.key}"
+            @logger.error "Message value: #{message.value}"
+          end
+        end
+
+      end
+
+      resp
     end
 
     def fetch_offsets(**options)

data/lib/kafka/client.rb

@@ -618,7 +618,15 @@ module Kafka
     #
     # @return [Array<String>] the list of topic names.
     def topics
-      @cluster.list_topics
+      attempts = 0
+      begin
+        attempts += 1
+        @cluster.list_topics
+      rescue Kafka::ConnectionError
+        @cluster.mark_as_stale!
+        retry unless attempts > 1
+        raise
+      end
     end
 
     # Lists all consumer groups in the cluster

data/lib/kafka/cluster.rb

@@ -397,6 +397,7 @@ module Kafka
     end
 
     def random_broker
+      refresh_metadata_if_necessary!
       node_id = cluster_info.brokers.sample.node_id
       connect_to_broker(node_id)
     end

data/lib/kafka/datadog.rb

@@ -40,7 +40,7 @@ module Kafka
       end
 
       def host
-        @host ||= ::Datadog::Statsd::DEFAULT_HOST
+        @host ||= default_host
       end
 
       def host=(host)
@@ -49,7 +49,7 @@ module Kafka
       end
 
       def port
-        @port ||= ::Datadog::Statsd::DEFAULT_PORT
+        @port ||= default_port
       end
 
       def port=(port)
@@ -77,6 +77,14 @@ module Kafka
 
       private
 
+      def default_host
+        ::Datadog::Statsd.const_defined?(:Connection) ? ::Datadog::Statsd::Connection::DEFAULT_HOST : ::Datadog::Statsd::DEFAULT_HOST
+      end
+
+      def default_port
+        ::Datadog::Statsd.const_defined?(:Connection) ? ::Datadog::Statsd::Connection::DEFAULT_PORT : ::Datadog::Statsd::DEFAULT_PORT
+      end
+
       def clear
         @statsd && @statsd.close
         @statsd = nil

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.1.beta2"
+  VERSION = "0.7.1.debugcorruption2"
 end

data/ruby-kafka.gemspec

@@ -41,7 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "extlz4"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "<4.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.1.beta2
+  version: 0.7.1.debugcorruption2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-11 00:00:00.000000000 Z
+date: 2018-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -201,7 +201,7 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 5.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -211,7 +211,7 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: statsd-ruby
   requirement: !ruby/object:Gem::Requirement