ruby-kafka 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c4c29bd5a3b7adccced77247c440446e9caa88cec568e71a5b75a7c72356f75
-  data.tar.gz: 38ed9f15e4384c1f0cc3c000a08075785d5a73779fa0ab0dafff3881328de9e7
+  metadata.gz: 63ec54e4024c83bb5a73583700391db7b89fed005bb4b196cf912424351bba0e
+  data.tar.gz: d567d51c5434f6ef034b769f78cc03286d91e875aa5ea9058623cd2e755cf28c
 SHA512:
-  metadata.gz: b84905cc979247733dd0d44056adcd876b6d5bdae264c1fa8b52fea9b04d3bd2f0c2dc3b4a22c875c228f5d6ef94cb4dff32e9e1f50f3d3a7ce19ad32615a479
-  data.tar.gz: b2de57d34f12c2e448383b7eb79263bdadc093a49b96e99bfbdda1db16566307702c09c4a18d38fd5452f1f5b63c27cf28f50c73c36b426b920ae6775ba56b6a
+  metadata.gz: 3f9f8269075c30eb60a3ee7afcca5c1fe8eab9aa6c5cf51c5512dc6b564299a5366d33bc97d08b61a17ecc4a299e066ecb48f86938c93643324a8083289ed3af
+  data.tar.gz: 92767b83a152bef74992d1b4dc4b1818bdce8715a4e1e32c53671eaf6ee8d28d8e76d4800bf141c8cdd6cc44ef51aef78335464dbe504f8f72245ec8fc964a9a

data/CHANGELOG.md

@@ -4,9 +4,13 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.5.5
+
+- Support PLAINTEXT and SSL URI schemes (#550).
+
 ## v0.5.4
 
-- Add support for config entries in the topic creation API.
+- Add support for config entries in the topic creation API (#540).
 - Don't fail on retry when the cluster is secured (#545).
 
 ## v0.5.3

data/README.md

@@ -2,7 +2,6 @@
 
 A Ruby client library for [Apache Kafka](http://kafka.apache.org/), a distributed log and message bus. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
 
-Although parts of this library work with Kafka 0.8 – specifically, the Producer API – it's being tested and developed against Kafka 0.9. The Consumer API is Kafka 0.9+ only.
 
 ## Table of Contents
 

data/lib/kafka/broker_uri.rb

@@ -0,0 +1,41 @@
+require "uri"
+
+module Kafka
+  module BrokerUri
+    DEFAULT_PORT = 9092
+    URI_SCHEMES = ["kafka", "kafka+ssl", "plaintext", "ssl"]
+
+    # Parses a Kafka broker URI string.
+    #
+    # Examples of valid strings:
+    # * `kafka1.something`
+    # * `kafka1.something:1234`
+    # * `kafka://kafka1.something:1234`
+    # * `kafka+ssl://kafka1.something:1234`
+    # * `plaintext://kafka1.something:1234`
+    #
+    # @param str [String] a Kafka broker URI string.
+    # @return [URI]
+    def self.parse(str)
+      # Make sure there's a scheme part if it's missing.
+      str = "kafka://" + str unless str.include?("://")
+
+      uri = URI.parse(str)
+      uri.port ||= DEFAULT_PORT
+
+      # Map some schemes to others.
+      case uri.scheme
+      when 'plaintext'
+        uri.scheme = 'kafka'
+      when 'ssl'
+        uri.scheme = 'kafka+ssl'
+      end
+
+      unless URI_SCHEMES.include?(uri.scheme)
+        raise Kafka::Error, "invalid protocol `#{uri.scheme}` in `#{str}`"
+      end
+
+      uri
+    end
+  end
+end

data/lib/kafka/client.rb

@@ -1,10 +1,9 @@
-require "openssl"
-require "uri"
-
+require "kafka/ssl_context"
 require "kafka/cluster"
 require "kafka/producer"
 require "kafka/consumer"
 require "kafka/heartbeat"
+require "kafka/broker_uri"
 require "kafka/async_producer"
 require "kafka/fetched_message"
 require "kafka/fetch_operation"
@@ -14,8 +13,6 @@ require "kafka/sasl_authenticator"
 
 module Kafka
   class Client
-    URI_SCHEMES = ["kafka", "kafka+ssl"]
-
     # Initializes a new Kafka client.
     #
     # @param seed_brokers [Array<String>, String] the list of brokers used to initialize
@@ -65,7 +62,13 @@ module Kafka
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
 
-      ssl_context = build_ssl_context(ssl_ca_cert_file_path, ssl_ca_cert, ssl_client_cert, ssl_client_cert_key, ssl_ca_certs_from_system)
+      ssl_context = SslContext.build(
+        ca_cert_file_path: ssl_ca_cert_file_path,
+        ca_cert: ssl_ca_cert,
+        client_cert: ssl_client_cert,
+        client_cert_key: ssl_client_cert_key,
+        ca_certs_from_system: ssl_ca_certs_from_system,
+      )
 
       sasl_authenticator = SaslAuthenticator.new(
         sasl_gssapi_principal: sasl_gssapi_principal,
@@ -605,55 +608,12 @@ module Kafka
       )
     end
 
-    def build_ssl_context(ca_cert_file_path, ca_cert, client_cert, client_cert_key, ssl_ca_certs_from_system)
-      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || ssl_ca_certs_from_system
-
-      ssl_context = OpenSSL::SSL::SSLContext.new
-
-      if client_cert && client_cert_key
-        ssl_context.set_params(
-          cert: OpenSSL::X509::Certificate.new(client_cert),
-          key: OpenSSL::PKey.read(client_cert_key)
-        )
-      elsif client_cert && !client_cert_key
-        raise ArgumentError, "Kafka client initialized with `ssl_client_cert` but no `ssl_client_cert_key`. Please provide both."
-      elsif !client_cert && client_cert_key
-        raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
-      end
-
-      if ca_cert || ca_cert_file_path || ssl_ca_certs_from_system
-        store = OpenSSL::X509::Store.new
-        Array(ca_cert).each do |cert|
-          store.add_cert(OpenSSL::X509::Certificate.new(cert))
-        end
-        if ca_cert_file_path
-          store.add_file(ca_cert_file_path)
-        end
-        if ssl_ca_certs_from_system
-          store.set_default_paths
-        end
-        ssl_context.cert_store = store
-      end
-
-      ssl_context
-    end
-
     def normalize_seed_brokers(seed_brokers)
       if seed_brokers.is_a?(String)
         seed_brokers = seed_brokers.split(",")
       end
 
-      seed_brokers.map do |connection|
-        connection = "kafka://" + connection unless connection =~ /:\/\//
-        uri = URI.parse(connection)
-        uri.port ||= 9092 # Default Kafka port.
-
-        unless URI_SCHEMES.include?(uri.scheme)
-          raise Kafka::Error, "invalid protocol `#{uri.scheme}` in `#{connection}`"
-        end
-
-        uri
-      end
+      seed_brokers.map {|str| BrokerUri.parse(str) }
     end
   end
 end

data/lib/kafka/cluster.rb

@@ -186,6 +186,11 @@ module Kafka
         @logger.warn "Leader not yet available for `#{name}`, waiting 1s..."
         sleep 1
 
+        retry
+      rescue Kafka::UnknownTopicOrPartition
+        @logger.warn "Topic `#{name}` not yet created, waiting 1s..."
+        sleep 1
+
         retry
       end
 

data/lib/kafka/ssl_context.rb

@@ -0,0 +1,39 @@
+require "openssl"
+
+module Kafka
+  module SslContext
+
+    def self.build(ca_cert_file_path: nil, ca_cert: nil, client_cert: nil, client_cert_key: nil, ca_certs_from_system: nil)
+      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key || ca_certs_from_system
+
+      ssl_context = OpenSSL::SSL::SSLContext.new
+
+      if client_cert && client_cert_key
+        ssl_context.set_params(
+          cert: OpenSSL::X509::Certificate.new(client_cert),
+          key: OpenSSL::PKey.read(client_cert_key)
+        )
+      elsif client_cert && !client_cert_key
+        raise ArgumentError, "Kafka client initialized with `ssl_client_cert` but no `ssl_client_cert_key`. Please provide both."
+      elsif !client_cert && client_cert_key
+        raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
+      end
+
+      if ca_cert || ca_cert_file_path || ca_certs_from_system
+        store = OpenSSL::X509::Store.new
+        Array(ca_cert).each do |cert|
+          store.add_cert(OpenSSL::X509::Certificate.new(cert))
+        end
+        if ca_cert_file_path
+          store.add_file(ca_cert_file_path)
+        end
+        if ca_certs_from_system
+          store.set_default_paths
+        end
+        ssl_context.cert_store = store
+      end
+
+      ssl_context
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.4"
+  VERSION = "0.5.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-16 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -310,6 +310,7 @@ files:
 - lib/kafka/async_producer.rb
 - lib/kafka/broker.rb
 - lib/kafka/broker_pool.rb
+- lib/kafka/broker_uri.rb
 - lib/kafka/client.rb
 - lib/kafka/cluster.rb
 - lib/kafka/compression.rb
@@ -382,6 +383,7 @@ files:
 - lib/kafka/sasl_authenticator.rb
 - lib/kafka/snappy_codec.rb
 - lib/kafka/socket_with_timeout.rb
+- lib/kafka/ssl_context.rb
 - lib/kafka/ssl_socket_with_timeout.rb
 - lib/kafka/statsd.rb
 - lib/kafka/version.rb