ruby-kafka 0.6.0.beta2 → 0.6.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8de7c8abb93903beb344e75576f60227f42e03b4fc05cfd535b69ed71443c20
-  data.tar.gz: e23592d1e5c00e145cba041729ba69ba80a6ed7452170d6fc0a04fbb472b42cd
+  metadata.gz: b0de3ad482f6130c080cc00b0dd7f4d862861838cfd6c762f6235defada9652f
+  data.tar.gz: c97c1432402dc5d78816ffb0c251e1c808c4ebaaad7609e3f8b7265d0661db8a
 SHA512:
-  metadata.gz: f3c7af12d2699a189d10e12117ecf06b315118fa373dc9365c50578dea3d5b1bc55f06401bd2bb044c778cf036b99daba9c4bc1a6cc6770eefa7c266ecbd0521
-  data.tar.gz: bfe265ae1b2af3024ffb10de3dc627cfda81b75916f7ade3d1537cc5ccae00dd6c134af70f5f6871be1a788e85ad2b4952db779588b59d17bd1bd1d9963b9dd4
+  metadata.gz: 647f4e5908073b08c597e90bc229c9962cf53121590297857caa641ebed7007331aa434238a719584b5c4ef66668a3a9fbe9a732105577b51f3fbc2539b6b3c9
+  data.tar.gz: dce76f84ed3ad8a010935321adc225f5ac757c33cfb39fee8d84c67c4f36bc4fdb053123f84638324695a4ec834d5765a0f44abd66eeb0c153762907f1737e55

data/README.md

@@ -37,6 +37,7 @@ A Ruby client library for [Apache Kafka](http://kafka.apache.org/), a distribute
     9. [Security](#security)
         1. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
         2. [Authentication using SASL](#authentication-using-sasl)
+    10. [Topic management](#topic-management)
 4. [Design](#design)
     1. [Producer Design](#producer-design)
     2. [Asynchronous Producer Design](#asynchronous-producer-design)
@@ -966,6 +967,77 @@ kafka = Kafka.new(
 )
 ```
 
+### Topic management
+
+In addition to producing and consuming messages, ruby-kafka supports managing Kafka topics and their configurations. See [the Kafka documentation](https://kafka.apache.org/documentation/#topicconfigs) for a full list of topic configuration keys.
+
+#### List all topics
+
+Return an array of topic names.
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.topics
+# => ["topic1", "topic2", "topic3"]
+```
+
+#### Create a topic
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.create_topic("topic")
+```
+
+By default, the new topic has 1 partition, replication factor 1 and default configs from the brokers. Those configurations are customizable:
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.create_topic("topic",
+  num_partitions: 3,
+  replication_factor: 2,
+  config: {
+    "max.message.bytes" => 100000
+  }
+)
+```
+
+#### Create more partitions for a topic
+
+After a topic is created, you can increase the number of partitions for the topic. The new number of partitions must be creater than the current one.
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.create_partitions_for("topic", num_partitions: 10)
+```
+
+#### Fetch configuration for a topic (alpha feature)
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.describe_topic("topic", ["max.message.bytes", "retention.ms"])
+# => {"max.message.bytes"=>"100000", "retention.ms"=>"604800000"}
+```
+
+#### Alter a topic configuration (alpha feature)
+
+Update the topic configurations.
+
+**NOTE**: This feature is for advanced usage. Only use this if you know what you're doing.
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.alter_topic("topic", "max.message.bytes" => 100000, "retention.ms" => 604800000)
+```
+
+#### Delete a topic
+
+```ruby
+kafka = Kafka.new(["kafka:9092"])
+kafka.delete_topic("topic")
+```
+
+After a topic is marked as deleted, Kafka only hides it from clients. It would take a while before a topic is completely deleted.
+
 ## Design
 
 The library has been designed as a layered system, with each layer having a clear responsibility:

data/lib/kafka/broker.rb

@@ -132,6 +132,12 @@ module Kafka
       send_request(request)
     end
 
+    def alter_configs(**options)
+      request = Protocol::AlterConfigsRequest.new(**options)
+
+      send_request(request)
+    end
+
     def create_partitions(**options)
       request = Protocol::CreatePartitionsRequest.new(**options)
 

data/lib/kafka/client.rb

@@ -520,6 +520,24 @@ module Kafka
       @cluster.describe_topic(name, configs)
     end
 
+    # Alter the configuration of a topic.
+    #
+    # Configuration keys must match
+    # [Kafka's topic-level configs](https://kafka.apache.org/documentation/#topicconfigs).
+    #
+    # @note This is an alpha level API and is subject to change.
+    #
+    # @example Describing the cleanup policy config of a topic
+    #   kafka = Kafka.new(["kafka1:9092"])
+    #   kafka.alter_topic("my-topic", "cleanup.policy" => "delete", "max.message.byte" => "100000")
+    #
+    # @param name [String] the name of the topic.
+    # @param configs [Hash<String, String>] hash of desired config keys and values.
+    # @return [nil]
+    def alter_topic(name, configs = {})
+      @cluster.alter_topic(name, configs)
+    end
+
     # Create partitions for a topic.
     #
     # @param name [String] the name of the topic.

data/lib/kafka/cluster.rb

@@ -235,6 +235,24 @@ module Kafka
       end
     end
 
+    def alter_topic(name, configs = {})
+      options = {
+        resources: [[Kafka::Protocol::RESOURCE_TYPE_TOPIC, name, configs]]
+      }
+
+      broker = controller_broker
+
+      @logger.info "Altering the config for topic `#{name}` using controller broker #{broker}"
+
+      response = broker.alter_configs(**options)
+
+      response.resources.each do |resource|
+        Protocol.handle_error(resource.error_code, resource.error_message)
+      end
+
+      nil
+    end
+
     def create_partitions_for(name, num_partitions:, timeout:)
       options = {
         topics: [[name, num_partitions, nil]],

data/lib/kafka/datadog.rb

@@ -349,5 +349,20 @@ module Kafka
 
       attach_to "async_producer.kafka"
     end
+
+    class FetcherSubscriber < StatsdSubscriber
+      def loop(event)
+        queue_size = event.payload.fetch(:queue_size)
+
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        gauge("fetcher.queue_size", queue_size, tags: tags)
+      end
+
+      attach_to "fetcher.kafka"
+    end
   end
 end

data/lib/kafka/fetcher.rb

@@ -75,6 +75,10 @@ module Kafka
     private
 
     def loop
+      @instrumenter.instrument("loop.fetcher", {
+        queue_size: @queue.size,
+      })
+
       if !@commands.empty?
         cmd, args = @commands.deq
 
@@ -99,6 +103,7 @@ module Kafka
 
     def handle_reset
       @next_offsets.clear
+      @queue.clear
     end
 
     def handle_stop(*)

data/lib/kafka/protocol.rb

@@ -28,6 +28,7 @@ module Kafka
     CREATE_TOPICS_API = 19
     DELETE_TOPICS_API = 20
     DESCRIBE_CONFIGS_API = 32
+    ALTER_CONFIGS_API = 33
     CREATE_PARTITIONS_API = 37
 
     # A mapping from numeric API keys to symbolic API names.
@@ -170,5 +171,7 @@ require "kafka/protocol/delete_topics_request"
 require "kafka/protocol/delete_topics_response"
 require "kafka/protocol/describe_configs_request"
 require "kafka/protocol/describe_configs_response"
+require "kafka/protocol/alter_configs_request"
+require "kafka/protocol/alter_configs_response"
 require "kafka/protocol/create_partitions_request"
 require "kafka/protocol/create_partitions_response"

data/lib/kafka/protocol/alter_configs_request.rb

@@ -0,0 +1,42 @@
+module Kafka
+  module Protocol
+
+    class AlterConfigsRequest
+      def initialize(resources:)
+        @resources = resources
+      end
+
+      def api_key
+        ALTER_CONFIGS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::AlterConfigsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@resources) do |type, name, configs|
+          encoder.write_int8(type)
+          encoder.write_string(name)
+
+          configs = configs.to_a
+          encoder.write_array(configs) do |config_name, config_value|
+            # Config value is nullable. In other cases, we must write the
+            # stringified value.
+            config_value = config_value.to_s unless config_value.nil?
+
+            encoder.write_string(config_name)
+            encoder.write_string(config_value)
+          end
+        end
+        # validate_only. We'll skip this feature.
+        encoder.write_boolean(false)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/alter_configs_response.rb

@@ -0,0 +1,47 @@
+module Kafka
+  module Protocol
+    class AlterConfigsResponse
+      class ResourceDescription
+        attr_reader :name, :type, :error_code, :error_message
+
+        def initialize(name:, type:, error_code:, error_message:)
+          @name = name
+          @type = type
+          @error_code = error_code
+          @error_message = error_message
+        end
+      end
+
+      attr_reader :resources
+
+      def initialize(throttle_time_ms:, resources:)
+        @throttle_time_ms = throttle_time_ms
+        @resources = resources
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        resources = decoder.array do
+          error_code = decoder.int16
+          error_message = decoder.string
+
+          resource_type = decoder.int8
+          if Kafka::Protocol::RESOURCE_TYPES[resource_type].nil?
+            raise Kafka::ProtocolError, "Resource type not supported: #{resource_type}"
+          end
+          resource_name = decoder.string
+
+          ResourceDescription.new(
+            type: RESOURCE_TYPES[resource_type],
+            name: resource_name,
+            error_code: error_code,
+            error_message: error_message
+          )
+        end
+
+        new(throttle_time_ms: throttle_time_ms, resources: resources)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/create_topics_request.rb

@@ -27,8 +27,8 @@ module Kafka
           # Replica assignments. We don't care.
           encoder.write_array([])
 
-          # Config entries. We don't care.
           encoder.write_array(config.fetch(:config)) do |config_name, config_value|
+            config_value = config_value.to_s unless config_value.nil?
             encoder.write_string(config_name)
             encoder.write_string(config_value)
           end

data/lib/kafka/statsd.rb

@@ -258,5 +258,17 @@ module Kafka
 
       attach_to "async_producer.kafka"
     end
+
+    class FetcherSubscriber < StatsdSubscriber
+      def loop(event)
+        queue_size = event.payload.fetch(:queue_size)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+
+        gauge("fetcher.#{client}.#{group_id}.queue_size", queue_size)
+      end
+
+      attach_to "fetcher.kafka"
+    end
   end
 end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.6.0.beta2"
+  VERSION = "0.6.0.beta3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0.beta2
+  version: 0.6.0.beta3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -336,6 +336,8 @@ files:
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb
+- lib/kafka/protocol/alter_configs_request.rb
+- lib/kafka/protocol/alter_configs_response.rb
 - lib/kafka/protocol/api_versions_request.rb
 - lib/kafka/protocol/api_versions_response.rb
 - lib/kafka/protocol/consumer_group_protocol.rb