ruby-kafka 0.7.10 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aa84a39ade791c3bce57152d426e95cb7d29575af3e8a9cb29fcded1c6cdad5
-  data.tar.gz: 46be738fb1a2b71bbbd5d5d3b2fc95b705aef5740f666095eeaf40c9a6980fc4
+  metadata.gz: 8ebe10cb2c93b4387a14d069c4291751808abed924cbee6f4ff2f00770aaa1c2
+  data.tar.gz: 1e392b4f2b5137cd83bd191b5c629fd98b41697d389fb5c68c3100d03c3ff218
 SHA512:
-  metadata.gz: 42e8059ab9b49e16a8e9c70e5739c9694d2142d5d440f8e2c18477cc151a24183536ae1548aed66a7e7ee6ea64c96ff2c9aacb63366ed92bd1612af6e52d8300
-  data.tar.gz: 5008c803cc61fbea993f7f2a22ca7ac3ebb448f91cbdca40d540b9f4604dab8b4e38fcf4b20006a16641b5239e78d6045ccbec70168171219b716090ea72ba09
+  metadata.gz: 037a8fe1a7495f5e3d723e047392f68a1373e91f8a00337beb7ae6e053d031d6ab1b8597767a1b8b43399850e23493295180a24250e0c2f09b4f626316cd6915
+  data.tar.gz: 108d7160593e452f27c1e828438f4cd8c2c6c009f081a0ff949ca330f9487b948aad73cf23ba39e12adb0ed9f773c667cfeebaad1625b33770057b7b69b3dcb2

data/.circleci/config.yml

@@ -203,6 +203,70 @@ jobs:
       - run: bundle install --path vendor/bundle
       - run: bundle exec rspec --profile --tag functional spec/functional
 
+  kafka-2.3:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.12-2.3.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.3.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.3.1
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec --profile --tag functional spec/functional
+
+  kafka-2.4:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.12-2.4.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.4.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.12-2.4.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec --profile --tag functional spec/functional
+
 workflows:
   version: 2
   test:
@@ -214,3 +278,5 @@ workflows:
       - kafka-2.0
       - kafka-2.1
       - kafka-2.2
+      - kafka-2.3
+      - kafka-2.4

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'Issue has been marked as stale due to a lack of activity.'
+        stale-pr-message: 'Pull request has been marked as stale due to a lack of activity.'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

data/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Changes and additions to the library will be listed here.
 
+## Unreleased
+
+## 1.0.0
+
+- Add client methods to manage configs (#759)
+- Support Kafka 2.3 and 2.4.
+
 ## 0.7.10
 
 - Fix logger again (#762)

data/README.md

@@ -113,6 +113,15 @@ Or install it yourself as:
     <td>Limited support</td>
     <td>Limited support</td>
   </tr>
+  <tr>
+    <th>Kafka 2.3</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
+    <th>Kafka 2.4</th>
+    <td>Limited support</td>
+    <td>Limited support</td>
+  </tr>
 </table>
 
 This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with the v0.5.x series. There's limited support for Kafka 0.8, and things should work with Kafka 0.11, although there may be performance issues due to changes in the protocol.
@@ -124,6 +133,9 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
 - **Kafka 1.0:** Everything that works with Kafka 0.11 should still work, but so far no features specific to Kafka 1.0 have been added.
 - **Kafka 2.0:** Everything that works with Kafka 1.0 should still work, but so far no features specific to Kafka 2.0 have been added.
 - **Kafka 2.1:** Everything that works with Kafka 2.0 should still work, but so far no features specific to Kafka 2.1 have been added.
+- **Kafka 2.2:** Everything that works with Kafka 2.1 should still work, but so far no features specific to Kafka 2.2 have been added.
+- **Kafka 2.3:** Everything that works with Kafka 2.2 should still work, but so far no features specific to Kafka 2.3 have been added.
+- **Kafka 2.4:** Everything that works with Kafka 2.3 should still work, but so far no features specific to Kafka 2.4 have been added.
 
 This library requires Ruby 2.1 or higher.
 

data/lib/kafka/async_producer.rb

@@ -103,6 +103,9 @@ module Kafka
     # @raise [BufferOverflow] if the message queue is full.
     # @return [nil]
     def produce(value, topic:, **options)
+      # We want to fail fast if `topic` isn't a String
+      topic = topic.to_str
+
       ensure_threads_running!
 
       if @queue.size >= @max_queue_size

data/lib/kafka/client.rb

@@ -138,6 +138,9 @@ module Kafka
     def deliver_message(value, key: nil, headers: {}, topic:, partition: nil, partition_key: nil, retries: 1)
       create_time = Time.now
 
+      # We want to fail fast if `topic` isn't a String
+      topic = topic.to_str
+
       message = PendingMessage.new(
         value: value,
         key: key,
@@ -337,6 +340,7 @@ module Kafka
     def consumer(
         group_id:,
         session_timeout: 30,
+        rebalance_timeout: 60,
         offset_commit_interval: 10,
         offset_commit_threshold: 0,
         heartbeat_interval: 10,
@@ -357,6 +361,7 @@ module Kafka
         logger: @logger,
         group_id: group_id,
         session_timeout: session_timeout,
+        rebalance_timeout: rebalance_timeout,
         retention_time: retention_time,
         instrumenter: instrumenter,
       )
@@ -530,6 +535,24 @@ module Kafka
       end
     end
 
+    # Describe broker configs
+    #
+    # @param broker_id [int] the id of the broker
+    # @param configs [Array] array of config keys.
+    # @return [Array<Kafka::Protocol::DescribeConfigsResponse::ConfigEntry>]
+    def describe_configs(broker_id, configs = [])
+      @cluster.describe_configs(broker_id, configs)
+    end
+
+    # Alter broker configs
+    #
+    # @param broker_id [int] the id of the broker
+    # @param configs [Array] array of config strings.
+    # @return [nil]
+    def alter_configs(broker_id, configs = [])
+      @cluster.alter_configs(broker_id, configs)
+    end
+
     # Creates a topic in the cluster.
     #
     # @example Creating a topic with log compaction
@@ -615,6 +638,14 @@ module Kafka
       @cluster.describe_group(group_id)
     end
 
+    # Fetch all committed offsets for a consumer group
+    #
+    # @param group_id [String] the id of the consumer group
+    # @return [Hash<String, Hash<Integer, Kafka::Protocol::OffsetFetchResponse::PartitionOffsetInfo>>]
+    def fetch_group_offsets(group_id)
+      @cluster.fetch_group_offsets(group_id)
+    end
+
     # Create partitions for a topic.
     #
     # @param name [String] the name of the topic.

data/lib/kafka/cluster.rb

@@ -45,6 +45,10 @@ module Kafka
         new_topics = topics - @target_topics
 
         unless new_topics.empty?
+          if new_topics.any? { |topic| topic.nil? or topic.empty? }
+            raise ArgumentError, "Topic must not be nil or empty"
+          end
+
           @logger.info "New topics added to target list: #{new_topics.to_a.join(', ')}"
 
           @target_topics.merge(new_topics)
@@ -139,6 +143,40 @@ module Kafka
       end
     end
 
+    def describe_configs(broker_id, configs = [])
+      options = {
+        resources: [[Kafka::Protocol::RESOURCE_TYPE_CLUSTER, broker_id.to_s, configs]]
+      }
+
+      info = cluster_info.brokers.find {|broker| broker.node_id == broker_id }
+      broker = @broker_pool.connect(info.host, info.port, node_id: info.node_id)
+
+      response = broker.describe_configs(**options)
+
+      response.resources.each do |resource|
+        Protocol.handle_error(resource.error_code, resource.error_message)
+      end
+
+      response.resources.first.configs
+    end
+
+    def alter_configs(broker_id, configs = [])
+      options = {
+        resources: [[Kafka::Protocol::RESOURCE_TYPE_CLUSTER, broker_id.to_s, configs]]
+      }
+
+      info = cluster_info.brokers.find {|broker| broker.node_id == broker_id }
+      broker = @broker_pool.connect(info.host, info.port, node_id: info.node_id)
+
+      response = broker.alter_configs(**options)
+
+      response.resources.each do |resource|
+        Protocol.handle_error(resource.error_code, resource.error_message)
+      end
+
+      nil
+    end
+
     def partitions_for(topic)
       add_target_topics([topic])
       refresh_metadata_if_necessary!
@@ -252,6 +290,20 @@ module Kafka
       group
     end
 
+    def fetch_group_offsets(group_id)
+      topics = get_group_coordinator(group_id: group_id)
+        .fetch_offsets(group_id: group_id, topics: nil)
+        .topics
+
+      topics.each do |_, partitions|
+        partitions.each do |_, response|
+          Protocol.handle_error(response.error_code)
+        end
+      end
+
+      topics
+    end
+
     def create_partitions_for(name, num_partitions:, timeout:)
       options = {
         topics: [[name, num_partitions, nil]],

data/lib/kafka/connection.rb

@@ -58,6 +58,9 @@ module Kafka
       @connect_timeout = connect_timeout || CONNECT_TIMEOUT
       @socket_timeout = socket_timeout || SOCKET_TIMEOUT
       @ssl_context = ssl_context
+
+      @socket = nil
+      @last_request = nil
     end
 
     def to_s

data/lib/kafka/consumer_group.rb

@@ -7,11 +7,12 @@ module Kafka
   class ConsumerGroup
     attr_reader :assigned_partitions, :generation_id, :group_id
 
-    def initialize(cluster:, logger:, group_id:, session_timeout:, retention_time:, instrumenter:)
+    def initialize(cluster:, logger:, group_id:, session_timeout:, rebalance_timeout:, retention_time:, instrumenter:)
       @cluster = cluster
       @logger = TaggedLogger.new(logger)
       @group_id = group_id
       @session_timeout = session_timeout
+      @rebalance_timeout = rebalance_timeout
       @instrumenter = instrumenter
       @member_id = ""
       @generation_id = nil
@@ -140,7 +141,9 @@ module Kafka
         response = coordinator.join_group(
           group_id: @group_id,
           session_timeout: @session_timeout,
+          rebalance_timeout: @rebalance_timeout,
           member_id: @member_id,
+          topics: @topics,
         )
 
         Protocol.handle_error(response.error_code)

data/lib/kafka/datadog.rb

@@ -40,7 +40,7 @@ module Kafka
       end
 
       def host
-        @host ||= default_host
+        @host
       end
 
       def host=(host)
@@ -49,7 +49,7 @@ module Kafka
       end
 
       def port
-        @port ||= default_port
+        @port
       end
 
       def port=(port)
@@ -77,14 +77,6 @@ module Kafka
 
       private
 
-      def default_host
-        ::Datadog::Statsd.const_defined?(:Connection) ? ::Datadog::Statsd::Connection::DEFAULT_HOST : ::Datadog::Statsd::DEFAULT_HOST
-      end
-
-      def default_port
-        ::Datadog::Statsd.const_defined?(:Connection) ? ::Datadog::Statsd::Connection::DEFAULT_PORT : ::Datadog::Statsd::DEFAULT_PORT
-      end
-
       def clear
         @statsd && @statsd.close
         @statsd = nil

data/lib/kafka/fetcher.rb

@@ -17,6 +17,9 @@ module Kafka
       @commands = Queue.new
       @next_offsets = Hash.new { |h, k| h[k] = {} }
 
+      # We are only running when someone calls start.
+      @running = false
+
       # Long poll until at least this many bytes can be fetched.
       @min_bytes = 1
 
@@ -110,7 +113,7 @@ module Kafka
       elsif @queue.size < @max_queue_size
         step
       else
-        @logger.warn "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
+        @logger.info "Reached max fetcher queue size (#{@max_queue_size}), sleeping 1s"
         sleep 1
       end
     ensure

data/lib/kafka/producer.rb

@@ -188,11 +188,14 @@ module Kafka
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
     def produce(value, key: nil, headers: {}, topic:, partition: nil, partition_key: nil, create_time: Time.now)
+      # We want to fail fast if `topic` isn't a String
+      topic = topic.to_str
+
       message = PendingMessage.new(
         value: value && value.to_s,
         key: key && key.to_s,
         headers: headers,
-        topic: topic.to_s,
+        topic: topic,
         partition: partition && Integer(partition),
         partition_key: partition_key && partition_key.to_s,
         create_time: create_time

data/lib/kafka/prometheus.rb

@@ -42,11 +42,11 @@ module Kafka
     class ConnectionSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @api_calls = Prometheus.registry.counter(:api_calls, 'Total calls')
-        @api_latency = Prometheus.registry.histogram(:api_latency, 'Latency', {}, LATENCY_BUCKETS)
-        @api_request_size = Prometheus.registry.histogram(:api_request_size, 'Request size', {}, SIZE_BUCKETS)
-        @api_response_size = Prometheus.registry.histogram(:api_response_size, 'Response size', {}, SIZE_BUCKETS)
-        @api_errors = Prometheus.registry.counter(:api_errors, 'Errors')
+        @api_calls = Prometheus.registry.counter(:api_calls, docstring: 'Total calls', labels: [:client, :api, :broker])
+        @api_latency = Prometheus.registry.histogram(:api_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :api, :broker])
+        @api_request_size = Prometheus.registry.histogram(:api_request_size, docstring: 'Request size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_response_size = Prometheus.registry.histogram(:api_response_size, docstring: 'Response size', buckets: SIZE_BUCKETS, labels: [:client, :api, :broker])
+        @api_errors = Prometheus.registry.counter(:api_errors, docstring: 'Errors', labels: [:client, :api, :broker])
       end
 
       def request(event)
@@ -58,34 +58,34 @@ module Kafka
         request_size = event.payload.fetch(:request_size, 0)
         response_size = event.payload.fetch(:response_size, 0)
 
-        @api_calls.increment(key)
-        @api_latency.observe(key, event.duration)
-        @api_request_size.observe(key, request_size)
-        @api_response_size.observe(key, response_size)
-        @api_errors.increment(key) if event.payload.key?(:exception)
+        @api_calls.increment(labels: key)
+        @api_latency.observe(event.duration, labels: key)
+        @api_request_size.observe(request_size, labels: key)
+        @api_response_size.observe(response_size, labels: key)
+        @api_errors.increment(labels: key) if event.payload.key?(:exception)
       end
     end
 
     class ConsumerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @process_messages = Prometheus.registry.counter(:consumer_process_messages, 'Total messages')
-        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, 'Total errors')
+        @process_messages = Prometheus.registry.counter(:consumer_process_messages, docstring: 'Total messages', labels: [:client, :group_id, :topic, :partition])
+        @process_message_errors = Prometheus.registry.counter(:consumer_process_message_errors, docstring: 'Total errors', labels: [:client, :group_id, :topic, :partition])
         @process_message_latency =
-          Prometheus.registry.histogram(:consumer_process_message_latency, 'Latency', {}, LATENCY_BUCKETS)
-        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, 'Offset lag')
-        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, 'Time lag of message')
-        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, 'Total errors in batch')
+          Prometheus.registry.histogram(:consumer_process_message_latency, docstring: 'Latency', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @offset_lag = Prometheus.registry.gauge(:consumer_offset_lag, docstring: 'Offset lag', labels: [:client, :group_id, :topic, :partition])
+        @time_lag = Prometheus.registry.gauge(:consumer_time_lag, docstring: 'Time lag of message', labels: [:client, :group_id, :topic, :partition])
+        @process_batch_errors = Prometheus.registry.counter(:consumer_process_batch_errors, docstring: 'Total errors in batch', labels: [:client, :group_id, :topic, :partition])
         @process_batch_latency =
-          Prometheus.registry.histogram(:consumer_process_batch_latency, 'Latency in batch', {}, LATENCY_BUCKETS)
-        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, 'Size of batch', {}, SIZE_BUCKETS)
-        @join_group = Prometheus.registry.histogram(:consumer_join_group, 'Time to join group', {}, DELAY_BUCKETS)
-        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, 'Total error in joining group')
-        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, 'Time to sync group', {}, DELAY_BUCKETS)
-        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, 'Total error in syncing group')
-        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, 'Time to leave group', {}, DELAY_BUCKETS)
-        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, 'Total error in leaving group')
-        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, 'Pause duration')
+          Prometheus.registry.histogram(:consumer_process_batch_latency, docstring: 'Latency in batch', buckets: LATENCY_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @batch_size = Prometheus.registry.histogram(:consumer_batch_size, docstring: 'Size of batch', buckets: SIZE_BUCKETS, labels: [:client, :group_id, :topic, :partition])
+        @join_group = Prometheus.registry.histogram(:consumer_join_group, docstring: 'Time to join group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @join_group_errors = Prometheus.registry.counter(:consumer_join_group_errors, docstring: 'Total error in joining group', labels: [:client, :group_id])
+        @sync_group = Prometheus.registry.histogram(:consumer_sync_group, docstring: 'Time to sync group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @sync_group_errors = Prometheus.registry.counter(:consumer_sync_group_errors, docstring: 'Total error in syncing group', labels: [:client, :group_id])
+        @leave_group = Prometheus.registry.histogram(:consumer_leave_group, docstring: 'Time to leave group', buckets: DELAY_BUCKETS, labels: [:client, :group_id])
+        @leave_group_errors = Prometheus.registry.counter(:consumer_leave_group_errors, docstring: 'Total error in leaving group', labels: [:client, :group_id])
+        @pause_duration = Prometheus.registry.gauge(:consumer_pause_duration, docstring: 'Pause duration', labels: [:client, :group_id, :topic, :partition])
       end
 
       def process_message(event)
@@ -102,18 +102,18 @@ module Kafka
         time_lag = create_time && ((Time.now - create_time) * 1000).to_i
 
         if event.payload.key?(:exception)
-          @process_message_errors.increment(key)
+          @process_message_errors.increment(labels: key)
         else
-          @process_message_latency.observe(key, event.duration)
-          @process_messages.increment(key)
+          @process_message_latency.observe(event.duration, labels: key)
+          @process_messages.increment(labels: key)
         end
 
-        @offset_lag.set(key, offset_lag)
+        @offset_lag.set(offset_lag, labels: key)
 
         # Not all messages have timestamps.
         return unless time_lag
 
-        @time_lag.set(key, time_lag)
+        @time_lag.set(time_lag, labels: key)
       end
 
       def process_batch(event)
@@ -126,10 +126,10 @@ module Kafka
         message_count = event.payload.fetch(:message_count)
 
         if event.payload.key?(:exception)
-          @process_batch_errors.increment(key)
+          @process_batch_errors.increment(labels: key)
         else
-          @process_batch_latency.observe(key, event.duration)
-          @process_messages.increment(key, message_count)
+          @process_batch_latency.observe(event.duration, labels: key)
+          @process_messages.increment(by: message_count, labels: key)
         end
       end
 
@@ -143,29 +143,29 @@ module Kafka
         offset_lag = event.payload.fetch(:offset_lag)
         batch_size = event.payload.fetch(:message_count)
 
-        @batch_size.observe(key, batch_size)
-        @offset_lag.set(key, offset_lag)
+        @batch_size.observe(batch_size, labels: key)
+        @offset_lag.set(offset_lag, labels: key)
       end
 
       def join_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @join_group.observe(key, event.duration)
+        @join_group.observe(event.duration, labels: key)
 
-        @join_group_errors.increment(key) if event.payload.key?(:exception)
+        @join_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def sync_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @sync_group.observe(key, event.duration)
+        @sync_group.observe(event.duration, labels: key)
 
-        @sync_group_errors.increment(key) if event.payload.key?(:exception)
+        @sync_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def leave_group(event)
         key = { client: event.payload.fetch(:client_id), group_id: event.payload.fetch(:group_id) }
-        @leave_group.observe(key, event.duration)
+        @leave_group.observe(event.duration, labels: key)
 
-        @leave_group_errors.increment(key) if event.payload.key?(:exception)
+        @leave_group_errors.increment(labels: key) if event.payload.key?(:exception)
       end
 
       def pause_status(event)
@@ -177,28 +177,28 @@ module Kafka
         }
 
         duration = event.payload.fetch(:duration)
-        @pause_duration.set(key, duration)
+        @pause_duration.set(duration, labels: key)
       end
     end
 
     class ProducerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, 'Produced messages total')
+        @produce_messages = Prometheus.registry.counter(:producer_produced_messages, docstring: 'Produced messages total', labels: [:client, :topic])
         @produce_message_size =
-          Prometheus.registry.histogram(:producer_message_size, 'Message size', {}, SIZE_BUCKETS)
-        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, 'Buffer size', {}, SIZE_BUCKETS)
-        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, 'Buffer fill ratio')
-        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, 'Buffer fill percentage')
-        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, 'Produce errors')
-        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, 'Deliver error')
+          Prometheus.registry.histogram(:producer_message_size, docstring: 'Message size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @buffer_size = Prometheus.registry.histogram(:producer_buffer_size, docstring: 'Buffer size', buckets: SIZE_BUCKETS, labels: [:client])
+        @buffer_fill_ratio = Prometheus.registry.histogram(:producer_buffer_fill_ratio, docstring: 'Buffer fill ratio', labels: [:client])
+        @buffer_fill_percentage = Prometheus.registry.histogram(:producer_buffer_fill_percentage, docstring: 'Buffer fill percentage', labels: [:client])
+        @produce_errors = Prometheus.registry.counter(:producer_produce_errors, docstring: 'Produce errors', labels: [:client, :topic])
+        @deliver_errors = Prometheus.registry.counter(:producer_deliver_errors, docstring: 'Deliver error', labels: [:client])
         @deliver_latency =
-          Prometheus.registry.histogram(:producer_deliver_latency, 'Delivery latency', {}, LATENCY_BUCKETS)
-        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, 'Total count of delivered messages')
-        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, 'Delivery attempts')
-        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, 'Ack')
-        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, 'Ack delay', {}, LATENCY_BUCKETS)
-        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, 'Ack errors')
+          Prometheus.registry.histogram(:producer_deliver_latency, docstring: 'Delivery latency', buckets: LATENCY_BUCKETS, labels: [:client])
+        @deliver_messages = Prometheus.registry.counter(:producer_deliver_messages, docstring: 'Total count of delivered messages', labels: [:client])
+        @deliver_attempts = Prometheus.registry.histogram(:producer_deliver_attempts, docstring: 'Delivery attempts', labels: [:client])
+        @ack_messages = Prometheus.registry.counter(:producer_ack_messages, docstring: 'Ack', labels: [:client, :topic])
+        @ack_delay = Prometheus.registry.histogram(:producer_ack_delay, docstring: 'Ack delay', buckets: LATENCY_BUCKETS, labels: [:client, :topic])
+        @ack_errors = Prometheus.registry.counter(:producer_ack_errors, docstring: 'Ack errors', labels: [:client, :topic])
       end
 
       def produce_message(event)
@@ -212,20 +212,20 @@ module Kafka
         buffer_fill_percentage = buffer_fill_ratio * 100.0
 
         # This gets us the write rate.
-        @produce_messages.increment(key)
-        @produce_message_size.observe(key, message_size)
+        @produce_messages.increment(labels: key)
+        @produce_message_size.observe(message_size, labels: key)
 
         # This gets us the avg/max buffer size per producer.
-        @buffer_size.observe({ client: client }, buffer_size)
+        @buffer_size.observe(buffer_size, labels: { client: client })
 
         # This gets us the avg/max buffer fill ratio per producer.
-        @buffer_fill_ratio.observe({ client: client }, buffer_fill_ratio)
-        @buffer_fill_percentage.observe({ client: client }, buffer_fill_percentage)
+        @buffer_fill_ratio.observe(buffer_fill_ratio, labels: { client: client })
+        @buffer_fill_percentage.observe(buffer_fill_percentage, labels: { client: client })
       end
 
       def buffer_overflow(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
-        @produce_errors.increment(key)
+        @produce_errors.increment(labels: key)
       end
 
       def deliver_messages(event)
@@ -233,40 +233,40 @@ module Kafka
         message_count = event.payload.fetch(:delivered_message_count)
         attempts = event.payload.fetch(:attempts)
 
-        @deliver_errors.increment(key) if event.payload.key?(:exception)
-        @deliver_latency.observe(key, event.duration)
+        @deliver_errors.increment(labels: key) if event.payload.key?(:exception)
+        @deliver_latency.observe(event.duration, labels: key)
 
         # Messages delivered to Kafka:
-        @deliver_messages.increment(key, message_count)
+        @deliver_messages.increment(by: message_count, labels: key)
 
         # Number of attempts to deliver messages:
-        @deliver_attempts.observe(key, attempts)
+        @deliver_attempts.observe(attempts, labels: key)
       end
 
       def ack_message(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
 
         # Number of messages ACK'd for the topic.
-        @ack_messages.increment(key)
+        @ack_messages.increment(labels: key)
 
         # Histogram of delay between a message being produced and it being ACK'd.
-        @ack_delay.observe(key, event.payload.fetch(:delay))
+        @ack_delay.observe(event.payload.fetch(:delay), labels: key)
       end
 
       def topic_error(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
 
-        @ack_errors.increment(key)
+        @ack_errors.increment(labels: key)
       end
     end
 
     class AsyncProducerSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, 'Queue size', {}, SIZE_BUCKETS)
-        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, 'Queue fill ratio')
-        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, 'Producer errors')
-        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, 'Dropped messages')
+        @queue_size = Prometheus.registry.histogram(:async_producer_queue_size, docstring: 'Queue size', buckets: SIZE_BUCKETS, labels: [:client, :topic])
+        @queue_fill_ratio = Prometheus.registry.histogram(:async_producer_queue_fill_ratio, docstring: 'Queue fill ratio', labels: [:client, :topic])
+        @produce_errors = Prometheus.registry.counter(:async_producer_produce_errors, docstring: 'Producer errors', labels: [:client, :topic])
+        @dropped_messages = Prometheus.registry.counter(:async_producer_dropped_messages, docstring: 'Dropped messages', labels: [:client])
       end
 
       def enqueue_message(event)
@@ -277,29 +277,28 @@ module Kafka
         queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
 
         # This gets us the avg/max queue size per producer.
-        @queue_size.observe(key, queue_size)
+        @queue_size.observe(queue_size, labels: key)
 
         # This gets us the avg/max queue fill ratio per producer.
-        @queue_fill_ratio.observe(key, queue_fill_ratio)
+        @queue_fill_ratio.observe(queue_fill_ratio, labels: key)
       end
 
       def buffer_overflow(event)
         key = { client: event.payload.fetch(:client_id), topic: event.payload.fetch(:topic) }
-        @produce_errors.increment(key)
+        @produce_errors.increment(labels: key)
       end
 
       def drop_messages(event)
         key = { client: event.payload.fetch(:client_id) }
         message_count = event.payload.fetch(:message_count)
-
-        @dropped_messages.increment(key, message_count)
+        @dropped_messages.increment(by: message_count, labels: key)
       end
     end
 
     class FetcherSubscriber < ActiveSupport::Subscriber
       def initialize
         super
-        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, 'Queue size')
+        @queue_size = Prometheus.registry.gauge(:fetcher_queue_size, docstring: 'Queue size', labels: [:client, :group_id])
       end
 
       def loop(event)
@@ -307,7 +306,7 @@ module Kafka
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
 
-        @queue_size.set({ client: client, group_id: group_id }, queue_size)
+        @queue_size.set(queue_size, labels: { client: client, group_id: group_id })
       end
     end
   end

data/lib/kafka/protocol/join_group_request.rb

@@ -7,13 +7,14 @@ module Kafka
     class JoinGroupRequest
       PROTOCOL_TYPE = "consumer"
 
-      def initialize(group_id:, session_timeout:, member_id:, topics: [])
+      def initialize(group_id:, session_timeout:, rebalance_timeout:, member_id:, topics: [])
         @group_id = group_id
         @session_timeout = session_timeout * 1000 # Kafka wants ms.
+        @rebalance_timeout = rebalance_timeout * 1000 # Kafka wants ms.
         @member_id = member_id || ""
         @protocol_type = PROTOCOL_TYPE
         @group_protocols = {
-          "standard" => ConsumerGroupProtocol.new(topics: ["test-messages"]),
+          "roundrobin" => ConsumerGroupProtocol.new(topics: topics),
         }
       end
 
@@ -21,6 +22,10 @@ module Kafka
         JOIN_GROUP_API
       end
 
+      def api_version
+        1
+      end
+
       def response_class
         JoinGroupResponse
       end
@@ -28,6 +33,7 @@ module Kafka
       def encode(encoder)
         encoder.write_string(@group_id)
         encoder.write_int32(@session_timeout)
+        encoder.write_int32(@rebalance_timeout)
         encoder.write_string(@member_id)
         encoder.write_string(@protocol_type)
 

data/lib/kafka/protocol/offset_fetch_request.rb

@@ -12,8 +12,10 @@ module Kafka
         OFFSET_FETCH_API
       end
 
+      # setting topics to nil fetches all offsets for a consumer group
+      # and that feature is only available in API version 2+
       def api_version
-        1
+        @topics.nil? ? 2 : 1
       end
 
       def response_class

data/lib/kafka/tagged_logger.rb

@@ -1,6 +1,7 @@
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'delegate'
 require 'logger'
 
 module Kafka

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.10"
+  VERSION = "1.0.0"
 end

data/ruby-kafka.gemspec

@@ -36,15 +36,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "extlz4"
   spec.add_development_dependency "zstd-ruby"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
-  spec.add_development_dependency "prometheus-client"
+  spec.add_development_dependency "prometheus-client", "~> 0.10.0"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.10
+  version: 1.0.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-07 00:00:00.000000000 Z
+date: 2020-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -128,14 +128,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: snappy
   requirement: !ruby/object:Gem::Requirement
@@ -212,7 +218,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -222,7 +228,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -244,16 +250,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +338,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".readygo"
 - ".rspec"