ruby-kafka 1.1.0.beta1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e5516c0741a1532bd320d8202e4cd23d18af5ddaf9581c6a39c69230b4c59e1
-  data.tar.gz: 66b47c941718a9a3a6e9f0523ec60dc6dd8b96ef2c82fda3571f7d150464cfeb
+  metadata.gz: 22b958f3d452f35c3c6084a5b8b790f370b96069e87ef5974838e5ea4b6945e1
+  data.tar.gz: eb6d704dbaace9c13a99bc93924d8a6b84ee5eb686c9c863a81d085ffeb7e92d
 SHA512:
-  metadata.gz: cb040bc7a76c13f131b59e2a15d9274542a4768595de221198b4e574aeb36ff5095ac14b0737b6a988cd4be7863f4bd44cccc7d2ad48ae2ee3f72f81256b08ca
-  data.tar.gz: 012474c8b6390eb2348906f320caa3759c12d8e9205713cd173f98a1463a432caa8c1c5cd700a2bea059e6601c5482b158d72af1fb199d3f9751f3cb1a3b5451
+  metadata.gz: 22c1d59bcdd42849849122b559f0d161653a5cfa492ffb36f28a875b109444d670a76bc62ba0856ae2ebce95fd10abce9e16834928fc7fe6eb2ee006827d307c
+  data.tar.gz: 3a51df6b1d40e1edbd96c06f3473319f0ac1b072b040b676a8e1d980fbaad114ad0248a7670cb7ae0a51a2c79ac390f2f4cfe8f65eedb5dd55be020c6011bc18

data/.circleci/config.yml

@@ -267,6 +267,38 @@ jobs:
       - run: bundle install --path vendor/bundle
       - run: bundle exec rspec --profile --tag functional spec/functional
 
+  kafka-2.5:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.12-2.5.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.5.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.5.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec --profile --tag functional spec/functional
+
 workflows:
   version: 2
   test:
@@ -280,3 +312,4 @@ workflows:
       - kafka-2.2
       - kafka-2.3
       - kafka-2.4
+      - kafka-2.5

data/CHANGELOG.md

@@ -6,6 +6,8 @@ Changes and additions to the library will be listed here.
 
 ## 1.1.0
 
+- Extra sanity checking when marking offsets as processed (#824).
+- Make `verify_hostname` settable for SSL contexts (#828).
 - Instrument `create_time` from last message in batch (#811).
 - Add client function for fetching topic replica count (#822).
 - Allow consumers to refresh the topic lists (#818).

data/README.md

@@ -118,10 +118,16 @@ Or install it yourself as:
     <td>Limited support</td>
     <td>Limited support</td>
   </tr>
+  <tr>
     <th>Kafka 2.4</th>
     <td>Limited support</td>
     <td>Limited support</td>
   </tr>
+  <tr>
+    <th>Kafka 2.5</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
 </table>
 
 This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.
@@ -136,6 +142,7 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
 - **Kafka 2.2:** Everything that works with Kafka 2.1 should still work, but so far no features specific to Kafka 2.2 have been added.
 - **Kafka 2.3:** Everything that works with Kafka 2.2 should still work, but so far no features specific to Kafka 2.3 have been added.
 - **Kafka 2.4:** Everything that works with Kafka 2.3 should still work, but so far no features specific to Kafka 2.4 have been added.
+- **Kafka 2.5:** Everything that works with Kafka 2.4 should still work, but so far no features specific to Kafka 2.5 have been added.
 
 This library requires Ruby 2.1 or higher.
 
@@ -945,6 +952,8 @@ This configures the store to look up CA certificates from the system default cer
 
 In order to authenticate the client to the cluster, you need to pass in a certificate and key created for the client and trusted by the brokers.
 
+**NOTE**: You can disable hostname validation by passing `verify_hostname: false`.
+
 ```ruby
 kafka = Kafka.new(
   ["kafka1:9092"],

data/lib/kafka/client.rb

@@ -65,6 +65,10 @@ module Kafka
     # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
     #   implements method token. See {Sasl::OAuth#initialize}
     #
+    # @param verify_hostname [Boolean, true] whether to verify that the host serving
+    #   the SSL certificate and the signing chain of the certificate have the correct domains
+    #   based on the CA certificate
+    #
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,

data/lib/kafka/consumer_group.rb

@@ -161,6 +161,12 @@ module Kafka
       @member_id = ""
       sleep 1
 
+      retry
+    rescue CoordinatorLoadInProgress
+      @logger.error "Coordinator broker still loading, retrying in 1s..."
+
+      sleep 1
+
       retry
     end
 

data/lib/kafka/datadog.rb

@@ -31,7 +31,7 @@ module Kafka
 
     class << self
       def statsd
-        @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+        @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags, socket_path: socket_path)
       end
 
       def statsd=(statsd)
@@ -57,6 +57,15 @@ module Kafka
         clear
       end
 
+      def socket_path
+        @socket_path
+      end
+
+      def socket_path=(socket_path)
+        @socket_path = socket_path
+        clear
+      end
+
       def namespace
         @namespace ||= STATSD_NAMESPACE
       end

data/lib/kafka/offset_manager.rb

@@ -50,9 +50,20 @@ module Kafka
     # @param offset [Integer] the offset of the message that should be marked as processed.
     # @return [nil]
     def mark_as_processed(topic, partition, offset)
-      @uncommitted_offsets += 1
+      unless @group.assigned_to?(topic, partition)
+        @logger.debug "Not marking #{topic}/#{partition}:#{offset} as processed for partition not assigned to this consumer."
+        return
+      end
       @processed_offsets[topic] ||= {}
 
+      last_processed_offset = @processed_offsets[topic][partition] || -1
+      if last_processed_offset > offset + 1
+        @logger.debug "Not overwriting newer offset #{topic}/#{partition}:#{last_processed_offset - 1} with older #{offset}"
+        return
+      end
+
+      @uncommitted_offsets += 1
+
       # The committed offset should always be the offset of the next message that the
       # application will read, thus adding one to the last message processed.
       @processed_offsets[topic][partition] = offset + 1

data/lib/kafka/ssl_context.rb

@@ -54,11 +54,12 @@ module Kafka
           store.set_default_paths
         end
         ssl_context.cert_store = store
-        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        # Verify certificate hostname if supported (ruby >= 2.4.0)
-        ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
       end
 
+      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
+
       ssl_context
     end
   end

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "1.1.0.beta1"
+  VERSION = "1.1.0"
 end

data/ruby-kafka.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
-  spec.license       = "Apache License Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.required_ruby_version = '>= 2.1.0'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 1.1.0.beta1
+  version: 1.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-05 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -476,7 +476,7 @@ files:
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
-- Apache License Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -489,9 +489,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6