ruby-kafka 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 045d5a61276b8499bdb55c05602625a44f558b38
-  data.tar.gz: 5192d0caa0e53cb995ff315aedbb35de61b214f0
+  metadata.gz: 48ff478bed517af54383f5751c8d2bff8e2d9af6
+  data.tar.gz: 5238160c645a2a491ec75292f2c1a62e99f0fac6
 SHA512:
-  metadata.gz: bfa82cc9074c2e8c3e8e82cd1dfe288fae912fc02a96d355ebcf69d0106746749c00b5120b6a7a151b5c8191ed499d52c1ee0543675772f7792d38d83d2b0ced
-  data.tar.gz: b03a4e3b7b50c6a0ec879c23a29059ced2f242ae8c5e6e6239afc9e9e083fcc0960b07b03b7ed9cd47c7a3f3c7855fc856448974b9ef5757f6500d741821edba
+  metadata.gz: eae662a727f7da9a533ccb9c7fa2054dc56187f60b1fed56f2c6e4b78cc56aa3e0e42b8c03bfa228ee754fdf5a9edff431314a5b40d6c2d585e30ad5d0a5f3e3
+  data.tar.gz: 34f00949dc40559aa6d0154e93dfa03e0d9db17834a55c2cdf7b03fced2b8c8993cbee6301852135e91e8f086efe99f859088720f42c4b733277df8b1420126f

data/.circleci/config.yml

@@ -23,18 +23,21 @@ jobs:
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:0.10.2.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:0.10.2.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
           KAFKA_PORT: 9094
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
     steps:
       - checkout
       - run: bundle install --path vendor/bundle
@@ -52,18 +55,19 @@ jobs:
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:0.11.0.1
         environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:0.11.0.1
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
           KAFKA_PORT: 9094
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
     steps:
       - checkout
       - run: bundle install --path vendor/bundle
@@ -81,18 +85,21 @@ jobs:
           KAFKA_ADVERTISED_PORT: 9092
           KAFKA_PORT: 9092
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:1.0.0
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9093
           KAFKA_PORT: 9093
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
       - image: wurstmeister/kafka:1.0.0
         environment:
           KAFKA_ADVERTISED_HOST_NAME: localhost
           KAFKA_ADVERTISED_PORT: 9094
           KAFKA_PORT: 9094
           KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
     steps:
       - checkout
       - run: bundle install --path vendor/bundle

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+- Add support for the topic deletion API (#528).
+- Add support for the partition creation API (#533).
+- Allow passing in the seed brokers in a positional argument (#538).
+
 ## v0.5.2
 
 - Instrument the start of message/batch processing (#496).

data/README.md

@@ -46,7 +46,7 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
 6. [Support and Discussion](#support-and-discussion)
 7. [Roadmap](#roadmap)
 8. [Higher level libraries](#higher-level-libraries)
-    1. [Message processing frameworks](#message-processing-framework)
+    1. [Message processing frameworks](#message-processing-frameworks)
     2. [Message publishing libraries](#message-publishing-libraries)
 
 ## Installation
@@ -74,13 +74,15 @@ Or install it yourself as:
     <th>Kafka 0.9</th>
     <th>Kafka 0.10</th>
     <th>Kafka 0.11</th>
+    <th>Kafka 1.0</th>
   </tr>
   <tr>
     <th>Producer API</th>
-    <td>Full support</td>
+    <td>Full support in v0.4.x</td>
     <td>Full support in v0.4.x</td>
     <td>Full support in v0.5.x</td>
     <td>Limited support</td>
+    <td>Limited support</td>
   </tr>
   <tr>
     <th>Consumer API</th>
@@ -88,6 +90,7 @@ Or install it yourself as:
     <td>Full support in v0.4.x</td>
     <td>Full support in v0.5.x</td>
     <td>Limited support</td>
+    <td>Limited support</td>
   </tr>
 </table>
 
@@ -97,6 +100,7 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
 - **Kafka 0.9:** Full support for the Producer and Consumer API in ruby-kafka v0.4.x.
 - **Kafka 0.10:** Full support for the Producer and Consumer API in ruby-kafka v0.5.x. Note that you _must_ run version 0.10.1 or higher of Kafka due to limitations in 0.10.0.
 - **Kafka 0.11:** Everything that works with Kafka 0.10 should still work, but so far no features specific to Kafka 0.11 have been added.
+- **Kafka 0.11:** Everything that works with Kafka 0.10 should still work, but so far no features specific to Kafka 1.0 have been added.
 
 This library requires Ruby 2.1 or higher.
 
@@ -111,13 +115,10 @@ A client must be initialized with at least one Kafka broker, from which the enti
 ```ruby
 require "kafka"
 
-kafka = Kafka.new(
-  # At least one of these nodes must be available:
-  seed_brokers: ["kafka1:9092", "kafka2:9092"],
-
-  # Set an optional client id in order to identify the client to Kafka:
-  client_id: "my-application",
-)
+# The first argument is a list of "seed brokers" that will be queried for the full
+# cluster topology. At least one of these *must* be available. `client_id` is
+# used to identify this client in logs and metrics. It's optional but recommended.
+kafka = Kafka.new(["kafka1:9092", "kafka2:9092"], client_id: "my-application")
 ```
 
 ### Producing Messages to Kafka
@@ -426,10 +427,7 @@ require "kafka"
 
 # Configure the Kafka client with the broker hosts and the Rails
 # logger.
-$kafka = Kafka.new(
-  seed_brokers: ["kafka1:9092", "kafka2:9092"],
-  logger: Rails.logger,
-)
+$kafka = Kafka.new(["kafka1:9092", "kafka2:9092"], logger: Rails.logger)
 
 # Set up an asynchronous producer that delivers its buffered messages
 # every ten seconds:
@@ -469,7 +467,7 @@ Consuming messages from a Kafka topic with ruby-kafka is simple:
 ```ruby
 require "kafka"
 
-kafka = Kafka.new(seed_brokers: ["kafka1:9092", "kafka2:9092"])
+kafka = Kafka.new(["kafka1:9092", "kafka2:9092"])
 
 kafka.each_message(topic: "greetings") do |message|
   puts message.offset, message.key, message.value
@@ -492,7 +490,7 @@ Using the API is simple:
 ```ruby
 require "kafka"
 
-kafka = Kafka.new(seed_brokers: ["kafka1:9092", "kafka2:9092"])
+kafka = Kafka.new(["kafka1:9092", "kafka2:9092"])
 
 # Consumers with the same group id will form a Consumer Group together.
 consumer = kafka.consumer(group_id: "my-consumer")
@@ -880,20 +878,30 @@ By enabling SSL encryption you can have some confidence that messages can be sen
 In this case you just need to pass a valid CA certificate as a string when configuring your `Kafka` client:
 
 ```ruby
-kafka = Kafka.new(
-  ssl_ca_cert: File.read('my_ca_cert.pem'),
-  # ...
-)
+kafka = Kafka.new(["kafka1:9092"], ssl_ca_cert: File.read('my_ca_cert.pem'))
 ```
 
 Without passing the CA certificate to the client it would be impossible to protect against [man-in-the-middle attacks](https://en.wikipedia.org/wiki/Man-in-the-middle_attack).
 
+##### Using your system's CA cert store
+
+If you want to use the CA certs from your system's default certificate store, you
+can use:
+
+```ruby
+kafka = Kafka.new(["kafka1:9092"], ssl_ca_certs_from_system: true)
+```
+
+This configures the store to look up CA certificates from the system default certificate store on an as needed basis. The location of the store can usually be determined by: 
+`OpenSSL::X509::DEFAULT_CERT_FILE`
+
 ##### Client Authentication
 
 In order to authenticate the client to the cluster, you need to pass in a certificate and key created for the client and trusted by the brokers.
 
 ```ruby
 kafka = Kafka.new(
+  ["kafka1:9092"],
   ssl_ca_cert: File.read('my_ca_cert.pem'),
   ssl_client_cert: File.read('my_client_cert.pem'),
   ssl_client_cert_key: File.read('my_client_cert_key.pem'),
@@ -916,6 +924,7 @@ In order to authenticate using GSSAPI, set your principal and optionally your ke
 
 ```ruby
 kafka = Kafka.new(
+  ["kafka1:9092"],
   sasl_gssapi_principal: 'kafka/kafka.example.com@EXAMPLE.COM',
   sasl_gssapi_keytab: '/etc/keytabs/kafka.keytab',
   # ...
@@ -927,6 +936,7 @@ In order to authenticate using PLAIN, you must set your username and password wh
 
 ```ruby
 kafka = Kafka.new(
+  ["kafka1:9092"],
   ssl_ca_cert: File.read('/etc/openssl/cert.pem'), # Optional but highly recommended
   sasl_plain_username: 'username',
   sasl_plain_password: 'password'
@@ -941,6 +951,7 @@ Since 0.11 kafka supports [SCRAM](https://kafka.apache.org/documentation.html#se
 
 ```ruby
 kafka = Kafka.new(
+  ["kafka1:9092"],
   sasl_scram_username: 'username',
   sasl_scram_password: 'password',
   sasl_scram_mechanism: 'sha256',

data/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     ports:
       - "2181:2181"
   kafka1:
-    image: wurstmeister/kafka:0.10.2.1
+    image: wurstmeister/kafka:0.11.0.1
     ports:
       - "9092:9092"
     environment:
@@ -16,7 +16,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
   kafka2:
-    image: wurstmeister/kafka:0.10.2.1
+    image: wurstmeister/kafka:0.11.0.1
     ports:
       - "9093:9092"
     environment:
@@ -27,7 +27,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
   kafka3:
-    image: wurstmeister/kafka:0.10.2.1
+    image: wurstmeister/kafka:0.11.0.1
     ports:
       - "9094:9092"
     environment:

data/examples/simple-consumer.rb

@@ -14,7 +14,7 @@ require "kafka"
 # with e.g. `$stderr` if you want to see what's happening under the hood.
 logger = Logger.new(StringIO.new)
 
-brokers = ENV.fetch("KAFKA_BROKERS")
+brokers = ENV.fetch("KAFKA_BROKERS").split(",")
 
 # Make sure to create this topic in your Kafka cluster or configure the
 # cluster to auto-create topics.

data/lib/kafka.rb

@@ -243,8 +243,14 @@ module Kafka
   #
   # @see Client#initialize
   # @return [Client]
-  def self.new(**options)
-    Client.new(**options)
+  def self.new(seed_brokers = nil, **options)
+    # We allow `seed_brokers` to be passed in either as a positional _or_ as a
+    # keyword argument.
+    if seed_brokers.nil?
+      Client.new(**options)
+    else
+      Client.new(seed_brokers: seed_brokers, **options)
+    end
   end
 end
 

data/lib/kafka/broker.rb

@@ -115,6 +115,24 @@ module Kafka
       send_request(request)
     end
 
+    def delete_topics(**options)
+      request = Protocol::DeleteTopicsRequest.new(**options)
+
+      send_request(request)
+    end
+
+    def describe_configs(**options)
+      request = Protocol::DescribeConfigsRequest.new(**options)
+
+      send_request(request)
+    end
+
+    def create_partitions(**options)
+      request = Protocol::CreatePartitionsRequest.new(**options)
+
+      send_request(request)
+    end
+
     def api_versions
       request = Protocol::ApiVersionsRequest.new
 

data/lib/kafka/client.rb

@@ -60,12 +60,12 @@ module Kafka
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
                    sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
                    sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
-                   sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil)
+                   sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil, ssl_ca_certs_from_system: false)
       @logger = logger || Logger.new(nil)
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
 
-      ssl_context = build_ssl_context(ssl_ca_cert_file_path, ssl_ca_cert, ssl_client_cert, ssl_client_cert_key)
+      ssl_context = build_ssl_context(ssl_ca_cert_file_path, ssl_ca_cert, ssl_client_cert, ssl_client_cert_key, ssl_ca_certs_from_system)
 
       sasl_authenticator = SaslAuthenticator.new(
         sasl_gssapi_principal: sasl_gssapi_principal,
@@ -463,6 +463,47 @@ module Kafka
       @cluster.create_topic(name, num_partitions: num_partitions, replication_factor: replication_factor, timeout: timeout)
     end
 
+    # Delete a topic in the cluster.
+    #
+    # @param name [String] the name of the topic.
+    # @param timeout [Integer] a duration of time to wait for the topic to be
+    #   completely marked deleted.
+    # @return [nil]
+    def delete_topic(name, timeout: 30)
+      @cluster.delete_topic(name, timeout: timeout)
+    end
+
+    # Describe the configuration of a topic.
+    #
+    # Retrieves the topic configuration from the Kafka brokers. Configuration names
+    # refer to [Kafka's topic-level configs](https://kafka.apache.org/documentation/#topicconfigs).
+    #
+    # @note This is an alpha level API and is subject to change.
+    #
+    # @example Describing the cleanup policy config of a topic
+    #   kafka = Kafka.new(["kafka1:9092"])
+    #   kafka.describe_topic("my-topic", ["cleanup.policy"])
+    #   #=> { "cleanup.policy" => "delete" }
+    #
+    # @param name [String] the name of the topic.
+    # @param configs [Array<String>] array of desired config names.
+    # @return [Hash<String, String>]
+    def describe_topic(name, configs = [])
+      @cluster.describe_topic(name, configs)
+    end
+
+    # Create partitions for a topic.
+    #
+    # @param name [String] the name of the topic.
+    # @param num_partitions [Integer] the number of desired partitions for
+    # the topic
+    # @param timeout [Integer] a duration of time to wait for the new
+    # partitions to be added.
+    # @return [nil]
+    def create_partitions_for(name, num_partitions: 1, timeout: 30)
+      @cluster.create_partitions_for(name, num_partitions: num_partitions, timeout: timeout)
+    end
+
     # Lists all topics in the cluster.
     #
     # @return [Array<String>] the list of topic names.
@@ -516,6 +557,15 @@ module Kafka
       }.to_h
     end
 
+    # Check whether current cluster supports a specific version or not
+    #
+    # @param api_key [Integer] API key.
+    # @param version [Integer] API version.
+    # @return [Boolean]
+    def supports_api?(api_key, version = nil)
+      @cluster.supports_api?(api_key, version)
+    end
+
     def apis
       @cluster.apis
     end
@@ -542,8 +592,8 @@ module Kafka
       )
     end
 
-    def build_ssl_context(ca_cert_file_path, ca_cert, client_cert, client_cert_key)
-      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key
+    def build_ssl_context(ca_cert_file_path, ca_cert, client_cert, client_cert_key, ssl_ca_certs_from_system)
+      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || ssl_ca_certs_from_system
 
       ssl_context = OpenSSL::SSL::SSLContext.new
 
@@ -558,7 +608,7 @@ module Kafka
         raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
       end
 
-      if ca_cert || ca_cert_file_path
+      if ca_cert || ca_cert_file_path || ssl_ca_certs_from_system
         store = OpenSSL::X509::Store.new
         Array(ca_cert).each do |cert|
           store.add_cert(OpenSSL::X509::Certificate.new(cert))
@@ -566,6 +616,9 @@ module Kafka
         if ca_cert_file_path
           store.add_file(ca_cert_file_path)
         end
+        if ssl_ca_certs_from_system
+          store.set_default_paths
+        end
         ssl_context.cert_store = store
       end
 

data/lib/kafka/cluster.rb

@@ -53,6 +53,17 @@ module Kafka
       apis.find {|api| api.api_key == api_key }
     end
 
+    def supports_api?(api_key, version = nil)
+      info = api_info(api_key)
+      if info.nil?
+        return false
+      elsif version.nil?
+        return true
+      else
+        return info.version_supported?(version)
+      end
+    end
+
     def apis
       @apis ||=
         begin
@@ -180,6 +191,64 @@ module Kafka
       @logger.info "Topic `#{name}` was created"
     end
 
+    def delete_topic(name, timeout:)
+      options = {
+        topics: [name],
+        timeout: timeout,
+      }
+
+      broker = controller_broker
+
+      @logger.info "Deleting topic `#{name}` using controller broker #{broker}"
+
+      response = broker.delete_topics(**options)
+
+      response.errors.each do |topic, error_code|
+        Protocol.handle_error(error_code)
+      end
+
+      @logger.info "Topic `#{name}` was deleted"
+    end
+
+    def describe_topic(name, configs = [])
+      options = {
+        resources: [[Kafka::Protocol::RESOURCE_TYPE_TOPIC, name, configs]]
+      }
+      broker = controller_broker
+
+      @logger.info "Fetching topic `#{name}`'s configs using controller broker #{broker}"
+
+      response = broker.describe_configs(**options)
+
+      response.resources.each do |resource|
+        Protocol.handle_error(resource.error_code, resource.error_message)
+      end
+      topic_description = response.resources.first
+      topic_description.configs.each_with_object({}) do |config, hash|
+        hash[config.name] = config.value
+      end
+    end
+
+    def create_partitions_for(name, num_partitions:, timeout:)
+      options = {
+        topics: [[name, num_partitions, nil]],
+        timeout: timeout
+      }
+
+      broker = controller_broker
+
+      @logger.info "Creating #{num_partitions} partition(s) for topic `#{name}` using controller broker #{broker}"
+
+      response = broker.create_partitions(**options)
+
+      response.errors.each do |topic, error_code, error_message|
+        Protocol.handle_error(error_code, error_message)
+      end
+      mark_as_stale!
+
+      @logger.info "Topic `#{name}` was updated"
+    end
+
     def resolve_offsets(topic, partitions, offset)
       add_target_topics([topic])
       refresh_metadata_if_necessary!
@@ -229,13 +298,17 @@ module Kafka
 
     def topics
       refresh_metadata_if_necessary!
-      cluster_info.topics.map(&:topic_name)
+      cluster_info.topics.select do |topic|
+        topic.topic_error_code == 0
+      end.map(&:topic_name)
     end
 
     # Lists all topics in the cluster.
     def list_topics
       response = random_broker.fetch_metadata(topics: nil)
-      response.topics.map(&:topic_name)
+      response.topics.select do |topic|
+        topic.topic_error_code == 0
+      end.map(&:topic_name)
     end
 
     def disconnect

data/lib/kafka/consumer.rb

@@ -20,7 +20,7 @@ module Kafka
   #
   #     require "kafka"
   #
-  #     kafka = Kafka.new(seed_brokers: ["kafka1:9092", "kafka2:9092"])
+  #     kafka = Kafka.new(["kafka1:9092", "kafka2:9092"])
   #
   #     # Create a new Consumer instance in the group `my-group`:
   #     consumer = kafka.consumer(group_id: "my-group")
@@ -371,7 +371,9 @@ module Kafka
 
       while @running
         begin
-          yield
+          @instrumenter.instrument("loop.consumer") do
+            yield
+          end
         rescue HeartbeatError, OffsetCommitError
           join_group
         rescue RebalanceInProgress

data/lib/kafka/consumer_group.rb

@@ -92,7 +92,7 @@ module Kafka
     end
 
     def heartbeat
-      @logger.info "Sending heartbeat..."
+      @logger.debug "Sending heartbeat..."
 
       response = coordinator.heartbeat(
         group_id: @group_id,

data/lib/kafka/producer.rb

@@ -14,7 +14,7 @@ module Kafka
   # do it for you, e.g.
   #
   #     # Will instantiate Kafka::Client
-  #     kafka = Kafka.new(...)
+  #     kafka = Kafka.new(["kafka1:9092", "kafka2:9092"])
   #
   #     # Will instantiate Kafka::Producer
   #     producer = kafka.producer
@@ -106,12 +106,7 @@ module Kafka
   #     # cluster to auto-create topics.
   #     topic = "random-messages"
   #
-  #     kafka = Kafka.new(
-  #       seed_brokers: brokers,
-  #       client_id: "simple-producer",
-  #       logger: logger,
-  #     )
-  #
+  #     kafka = Kafka.new(brokers, client_id: "simple-producer", logger: logger)
   #     producer = kafka.producer
   #
   #     begin

data/lib/kafka/protocol.rb

@@ -26,6 +26,9 @@ module Kafka
     SASL_HANDSHAKE_API = 17
     API_VERSIONS_API = 18
     CREATE_TOPICS_API = 19
+    DELETE_TOPICS_API = 20
+    DESCRIBE_CONFIGS_API = 32
+    CREATE_PARTITIONS_API = 37
 
     # A mapping from numeric API keys to symbolic API names.
     APIS = {
@@ -43,6 +46,9 @@ module Kafka
       SASL_HANDSHAKE_API => :sasl_handshake,
       API_VERSIONS_API => :api_versions,
       CREATE_TOPICS_API => :create_topics,
+      DELETE_TOPICS_API => :delete_topics,
+      DESCRIBE_CONFIGS_API => :describe_configs_api,
+      CREATE_PARTITIONS_API => :create_partitions
     }
 
     # A mapping from numeric error codes to exception classes.
@@ -87,19 +93,38 @@ module Kafka
       42 => InvalidRequest
     }
 
+    # A mapping from int to corresponding resource type in symbol.
+    # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
+    RESOURCE_TYPE_UNKNOWN = 0
+    RESOURCE_TYPE_ANY = 1
+    RESOURCE_TYPE_TOPIC = 2
+    RESOURCE_TYPE_GROUP = 3
+    RESOURCE_TYPE_CLUSTER = 4
+    RESOURCE_TYPE_TRANSACTIONAL_ID = 5
+    RESOURCE_TYPE_DELEGATION_TOKEN = 6
+    RESOURCE_TYPES = {
+      RESOURCE_TYPE_UNKNOWN => :unknown,
+      RESOURCE_TYPE_ANY => :any,
+      RESOURCE_TYPE_TOPIC => :topic,
+      RESOURCE_TYPE_GROUP => :group,
+      RESOURCE_TYPE_CLUSTER => :cluster,
+      RESOURCE_TYPE_TRANSACTIONAL_ID => :transactional_id,
+      RESOURCE_TYPE_DELEGATION_TOKEN => :delegation_token,
+    }
+
     # Handles an error code by either doing nothing (if there was no error) or
     # by raising an appropriate exception.
     #
     # @param error_code Integer
     # @raise [ProtocolError]
     # @return [nil]
-    def self.handle_error(error_code)
+    def self.handle_error(error_code, error_message = nil)
       if error_code == 0
         # No errors, yay!
       elsif error = ERRORS[error_code]
-        raise error
+        raise error, error_message
       else
-        raise UnknownError, "Unknown error with code #{error_code}"
+        raise UnknownError, "Unknown error with code #{error_code} #{error_message}"
       end
     end
 
@@ -141,3 +166,9 @@ require "kafka/protocol/sasl_handshake_request"
 require "kafka/protocol/sasl_handshake_response"
 require "kafka/protocol/create_topics_request"
 require "kafka/protocol/create_topics_response"
+require "kafka/protocol/delete_topics_request"
+require "kafka/protocol/delete_topics_response"
+require "kafka/protocol/describe_configs_request"
+require "kafka/protocol/describe_configs_response"
+require "kafka/protocol/create_partitions_request"
+require "kafka/protocol/create_partitions_response"

data/lib/kafka/protocol/api_versions_response.rb

@@ -13,6 +13,10 @@ module Kafka
           Protocol.api_name(api_key)
         end
 
+        def version_supported?(version)
+          (min_version..max_version).include?(version)
+        end
+
         def to_s
           "#{api_name}=#{min_version}..#{max_version}"
         end

data/lib/kafka/protocol/create_partitions_request.rb

@@ -0,0 +1,40 @@
+module Kafka
+  module Protocol
+
+    class CreatePartitionsRequest
+      def initialize(topics:, timeout:)
+        @topics, @timeout = topics, timeout
+      end
+
+      def api_key
+        CREATE_PARTITIONS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::CreatePartitionsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@topics) do |topic, count, assignments|
+          encoder.write_string(topic)
+          encoder.write_int32(count)
+          encoder.write_array(assignments) do |assignment|
+            encoder.write_array(assignment) do |broker|
+              encoder.write_int32(broker)
+            end
+          end
+        end
+        # Timeout is in ms.
+        encoder.write_int32(@timeout * 1000)
+        # validate_only. There isn't any use case for this in real life. So
+        # let's ignore it for now
+        encoder.write_boolean(false)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/create_partitions_response.rb

@@ -0,0 +1,26 @@
+module Kafka
+  module Protocol
+
+    class CreatePartitionsResponse
+      attr_reader :errors
+
+      def initialize(throttle_time_ms:, errors:)
+        @throttle_time_ms = throttle_time_ms
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        errors = decoder.array do
+          topic = decoder.string
+          error_code = decoder.int16
+          error_message = decoder.string
+          [topic, error_code, error_message]
+        end
+
+        new(throttle_time_ms: throttle_time_ms, errors: errors)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/delete_topics_request.rb

@@ -0,0 +1,31 @@
+module Kafka
+  module Protocol
+
+    class DeleteTopicsRequest
+      def initialize(topics:, timeout:)
+        @topics, @timeout = topics, timeout
+      end
+
+      def api_key
+        DELETE_TOPICS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::DeleteTopicsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@topics) do |topic|
+          encoder.write_string(topic)
+        end
+        # Timeout is in ms.
+        encoder.write_int32(@timeout * 1000)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/delete_topics_response.rb

@@ -0,0 +1,24 @@
+module Kafka
+  module Protocol
+
+    class DeleteTopicsResponse
+      attr_reader :errors
+
+      def initialize(errors:)
+        @errors = errors
+      end
+
+      def self.decode(decoder)
+        errors = decoder.array do
+          topic = decoder.string
+          error_code = decoder.int16
+
+          [topic, error_code]
+        end
+
+        new(errors: errors)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/describe_configs_request.rb

@@ -0,0 +1,33 @@
+module Kafka
+  module Protocol
+
+    class DescribeConfigsRequest
+      def initialize(resources:)
+        @resources = resources
+      end
+
+      def api_key
+        DESCRIBE_CONFIGS_API
+      end
+
+      def api_version
+        0
+      end
+
+      def response_class
+        Protocol::DescribeConfigsResponse
+      end
+
+      def encode(encoder)
+        encoder.write_array(@resources) do |type, name, configs|
+          encoder.write_int8(type)
+          encoder.write_string(name)
+          encoder.write_array(configs) do |config|
+            encoder.write_string(config)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/describe_configs_response.rb

@@ -0,0 +1,71 @@
+module Kafka
+  module Protocol
+    class DescribeConfigsResponse
+      class ResourceDescription
+        attr_reader :name, :type, :error_code, :error_message, :configs
+
+        def initialize(name:, type:, error_code:, error_message:, configs:)
+          @name = name
+          @type = type
+          @error_code = error_code
+          @error_message = error_message
+          @configs = configs
+        end
+      end
+
+      class ConfigEntry
+        attr_reader :name, :value, :read_only, :is_default, :is_sensitive
+
+        def initialize(name:, value:, read_only:, is_default:, is_sensitive:)
+          @name = name
+          @value = value
+          @read_only = read_only
+          @is_default = is_default
+          @is_sensitive = is_sensitive
+        end
+      end
+
+      attr_reader :resources
+
+      def initialize(throttle_time_ms:, resources:)
+        @throttle_time_ms = throttle_time_ms
+        @resources = resources
+      end
+
+      def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+        resources = decoder.array do
+          error_code = decoder.int16
+          error_message = decoder.string
+
+          resource_type = decoder.int8
+          if Kafka::Protocol::RESOURCE_TYPES[resource_type].nil?
+            raise Kafka::ProtocolError, "Resource type not supported: #{resource_type}"
+          end
+          resource_name = decoder.string
+
+          configs = decoder.array do
+            ConfigEntry.new(
+              name: decoder.string,
+              value: decoder.string,
+              read_only: decoder.boolean,
+              is_default: decoder.boolean,
+              is_sensitive: decoder.boolean,
+            )
+          end
+
+          ResourceDescription.new(
+            type: RESOURCE_TYPES[resource_type],
+            name: resource_name,
+            error_code: error_code,
+            error_message: error_message,
+            configs: configs
+          )
+        end
+
+        new(throttle_time_ms: throttle_time_ms, resources: resources)
+      end
+    end
+
+  end
+end

data/lib/kafka/protocol/encoder.rb

@@ -30,7 +30,7 @@ module Kafka
       # @param boolean [Boolean]
       # @return [nil]
       def write_boolean(boolean)
-        write(boolean ? 0x1 : 0x0)
+        boolean ? write_int8(1) : write_int8(0)
       end
 
       # Writes an 8-bit integer to the IO object.

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-09 00:00:00.000000000 Z
+date: 2018-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -299,9 +299,6 @@ files:
 - benchmarks/message_encoding.rb
 - bin/console
 - bin/setup
-- ci/consumer.rb
-- ci/init.rb
-- ci/producer.rb
 - docker-compose.yml
 - examples/consumer-group.rb
 - examples/firehose-consumer.rb
@@ -340,9 +337,15 @@ files:
 - lib/kafka/protocol/api_versions_request.rb
 - lib/kafka/protocol/api_versions_response.rb
 - lib/kafka/protocol/consumer_group_protocol.rb
+- lib/kafka/protocol/create_partitions_request.rb
+- lib/kafka/protocol/create_partitions_response.rb
 - lib/kafka/protocol/create_topics_request.rb
 - lib/kafka/protocol/create_topics_response.rb
 - lib/kafka/protocol/decoder.rb
+- lib/kafka/protocol/delete_topics_request.rb
+- lib/kafka/protocol/delete_topics_response.rb
+- lib/kafka/protocol/describe_configs_request.rb
+- lib/kafka/protocol/describe_configs_response.rb
 - lib/kafka/protocol/encoder.rb
 - lib/kafka/protocol/fetch_request.rb
 - lib/kafka/protocol/fetch_response.rb
@@ -383,7 +386,6 @@ files:
 - lib/kafka/statsd.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
-- performance/profile.rb
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:

data/ci/consumer.rb

@@ -1,18 +0,0 @@
-# Consumes messages from a Kafka topic.
-
-require_relative "init"
-
-consumer = $kafka.consumer(group_id: "greetings-group")
-consumer.subscribe("greetings")
-
-num_messages = 0
-
-trap("TERM") { consumer.stop }
-
-consumer.each_message do |message|
-  num_messages += 1
-
-  if num_messages % 1000 == 0
-    puts "Processed #{num_messages} messages"
-  end
-end

data/ci/init.rb

@@ -1,17 +0,0 @@
-$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
-
-require "kafka"
-
-logger = Logger.new(STDOUT)
-logger.level = Logger::INFO
-logger.formatter = ->(_, _, _, msg) { msg }
-
-STDOUT.sync = true
-
-$kafka = Kafka.new(
-  logger: logger,
-  seed_brokers: ENV.fetch("HEROKU_KAFKA_URL"),
-  ssl_ca_cert: ENV.fetch("HEROKU_KAFKA_TRUSTED_CERT"),
-  ssl_client_cert: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT"),
-  ssl_client_cert_key: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT_KEY"),
-)

data/ci/producer.rb

@@ -1,25 +0,0 @@
-# Continuously produces messages to a Kafka topic.
-
-require_relative "init"
-
-producer = $kafka.async_producer(
-  delivery_interval: 1,
-  max_queue_size: 5_000,
-  max_buffer_size: 10_000,
-)
-
-num_messages = 0
-shutdown = false
-
-trap("TERM") { shutdown = true }
-
-until shutdown
-  begin
-    producer.produce("hello", key: "world", topic: "greetings")
-  rescue Kafka::BufferOverflow
-    puts "Buffer overflow, backing off..."
-    sleep 10
-  end
-end
-
-producer.shutdown

data/performance/profile.rb

@@ -1,39 +0,0 @@
-$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
-$LOAD_PATH.unshift(File.expand_path("../../spec", __FILE__))
-
-require "kafka"
-require "ruby-prof"
-require "dotenv"
-require "test_cluster"
-
-Dotenv.load
-
-# Number of times do iterate.
-N = 10_000
-
-KAFKA_CLUSTER = TestCluster.new
-KAFKA_CLUSTER.start
-
-logger = Logger.new(nil)
-
-kafka = Kafka.new(
-  seed_brokers: KAFKA_CLUSTER.kafka_hosts,
-  client_id: "test",
-  logger: logger,
-)
-
-producer = kafka.producer(
-  max_buffer_size: 100_000,
-)
-
-RubyProf.start
-
-N.times do
-  producer.produce("hello", topic: "greetings")
-end
-
-result = RubyProf.stop
-printer = RubyProf::FlatPrinter.new(result)
-printer.print(STDOUT)
-
-KAFKA_CLUSTER.stop