ruby-kafka 0.3.18.beta2 → 0.4.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a315e2a5db26fa2430705e5dc25757593682703b
-  data.tar.gz: d80c0b9f184d4ec2da61139de39bf97177c7de4a
+  metadata.gz: 9ac9f20c365f90e85bd6df6000838fbc349b893b
+  data.tar.gz: fc16d9bf3784a21d1a82ec106b5b528bf57bfbe2
 SHA512:
-  metadata.gz: 63f090c16636aff10749d7e20628996207bf07dcd68da7fc58a76f49113972eb35e995b46af91d4b418692effefdc737e652750a583b6d7c3f066d5e797ff1e6
-  data.tar.gz: d5e813fbdf4d9663ca7e7718493f6b6b34523828eefd1697f69da2fb715fa1b533f84681be2fbd38ecd023c9b70bfb5085b04873764ab6f3cc040cc95c2f1ef0
+  metadata.gz: 22c77ea79b8e971916e1671253b709b207501a3e068d29e5629efea30733e9ab224748f068cc916c6472f7507c2e1073f7b0a3e9ee2cdbcafee21bd27e21f277
+  data.tar.gz: e5ecd931e2348ee989bcaf70387b30e64d9b24560231745b1c4d0e275cfcc0f65202cdf018b45bcc1678699ac05648bb4f0a5e5689eee030958823a864e73fb1

data/CHANGELOG.md

@@ -4,6 +4,12 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.4.0
+
+- Support SASL authentication (#334 and #370)
+- Allow loading SSL certificates from files (#371)
+- Add Statsd metric reporting (#373)
+
 ## v0.3.17
 
 - Re-commit previously committed offsets periodically with an interval of half

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ruby-kafka (0.3.18.beta1)
+    ruby-kafka (0.3.18.beta2)
       gssapi (>= 1.2.0)
 
 GEM
@@ -58,6 +58,7 @@ GEM
     ruby-prof (0.15.9)
     slop (3.6.0)
     snappy (0.0.12)
+    statsd-ruby (1.4.0)
     thread_safe (0.3.5)
     timecop (0.8.0)
     tzinfo (1.2.2)
@@ -81,6 +82,7 @@ DEPENDENCIES
   ruby-kafka!
   ruby-prof
   snappy
+  statsd-ruby
   timecop
 
 RUBY VERSION

data/README.md

@@ -31,7 +31,8 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
     5. [Logging](#logging)
     6. [Instrumentation](#instrumentation)
     7. [Monitoring](#monitoring)
-        1. [Reporting Metrics to Datadog](#reporting-metrics-to-datadog)
+        1. [Reporting Metrics to Statsd](#reporting-metrics-to-statsd)
+        2. [Reporting Metrics to Datadog](#reporting-metrics-to-datadog)
     8. [Understanding Timeouts](#understanding-timeouts)
     9. [Security](#security)
         1. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
@@ -732,7 +733,25 @@ It is highly recommended that you monitor your Kafka client applications in prod
 * consumer processing errors, indicating exceptions are being raised in the processing code;
 * frequent consumer rebalances, which may indicate unstable network conditions or consumer configurations.
 
-You can quite easily build monitoring on top of the provided [instrumentation hooks](#instrumentation). In order to further help with monitoring, a prebuilt [Datadog](https://www.datadoghq.com/) reporter is included with ruby-kafka.
+You can quite easily build monitoring on top of the provided [instrumentation hooks](#instrumentation). In order to further help with monitoring, a prebuilt [Statsd](https://github.com/etsy/statsd) and [Datadog](https://www.datadoghq.com/) reporter is included with ruby-kafka.
+
+
+#### Reporting Metrics to Statsd
+
+The Statsd reporter is automatically enabled when the `kafka/statsd` library is required. You can optionally change the configuration.
+
+```ruby
+require "kafka/statds"
+
+# Default is "ruby_kafka".
+Kafka::Statsd.namespace = "custom-namespace"
+
+# Default is "127.0.0.1".
+Kafka::Statsd.host = "statsd.something.com"
+
+# Default is 8125.
+Kafka::Statsd.port = 1234
+```
 
 
 #### Reporting Metrics to Datadog
@@ -815,9 +834,9 @@ Typically, Kafka certificates come in the JKS format, which isn't supported by r
 
 #### Authentication using SASL
 
-Kafka has support for using SASL to authenticate clients. Currently only the GSSAPI mechanism is supported by ruby-kafka.
+Kafka has support for using SASL to authenticate clients. Currently GSSAPI and PLAIN mechanisms are supported by ruby-kafka.
 
-In order to authenticate using SASL, set your principal and optionally your keytab when initializing the Kafka client:
+In order to authenticate using GSSAPI, set your principal and optionally your keytab when initializing the Kafka client:
 
 ```ruby
 kafka = Kafka.new(
@@ -827,6 +846,19 @@ kafka = Kafka.new(
 )
 ```
 
+In order to authenticate using PLAIN, you must set your username and password when initializing the Kafka client:
+
+```ruby
+kafka = Kafka.new(
+  ssl_ca_cert: File.read('/etc/openssl/cert.pem'), # Optional but highly recommended
+  sasl_plain_username: 'username',
+  sasl_plain_password: 'password'
+  # ...
+)
+```
+
+**NOTE**: It is __highly__ recommended that you use SSL for encryption when using SASL_PLAIN
+
 ## Design
 
 The library has been designed as a layered system, with each layer having a clear responsibility:

data/lib/kafka/client.rb

@@ -34,6 +34,9 @@ module Kafka
     # @param ssl_ca_cert [String, Array<String>, nil] a PEM encoded CA cert, or an Array of
     #   PEM encoded CA certs, to use with an SSL connection.
     #
+    # @param ssl_ca_cert_file_path [String, nil] a path on the filesystem to a PEM encoded CA cert
+    #   to use with an SSL connection.
+    #
     # @param ssl_client_cert [String, nil] a PEM encoded client cert to use with an
     #   SSL connection. Must be used in combination with ssl_client_cert_key.
     #
@@ -46,13 +49,14 @@ module Kafka
     #
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
-                   ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
-                   sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil)
+                   ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
+                   sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
+                   sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil)
       @logger = logger || Logger.new(nil)
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
 
-      ssl_context = build_ssl_context(ssl_ca_cert, ssl_client_cert, ssl_client_cert_key)
+      ssl_context = build_ssl_context(ssl_ca_cert_file_path, ssl_ca_cert, ssl_client_cert, ssl_client_cert_key)
 
       @connection_builder = ConnectionBuilder.new(
         client_id: client_id,
@@ -62,7 +66,10 @@ module Kafka
         logger: @logger,
         instrumenter: @instrumenter,
         sasl_gssapi_principal: sasl_gssapi_principal,
-        sasl_gssapi_keytab: sasl_gssapi_keytab
+        sasl_gssapi_keytab: sasl_gssapi_keytab,
+        sasl_plain_authzid: sasl_plain_authzid,
+        sasl_plain_username: sasl_plain_username,
+        sasl_plain_password: sasl_plain_password
       )
 
       @cluster = initialize_cluster
@@ -464,8 +471,8 @@ module Kafka
       )
     end
 
-    def build_ssl_context(ca_cert, client_cert, client_cert_key)
-      return nil unless ca_cert || client_cert || client_cert_key
+    def build_ssl_context(ca_cert_file_path, ca_cert, client_cert, client_cert_key)
+      return nil unless ca_cert_file_path || ca_cert || client_cert || client_cert_key
 
       ssl_context = OpenSSL::SSL::SSLContext.new
 
@@ -480,11 +487,14 @@ module Kafka
         raise ArgumentError, "Kafka client initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both."
       end
 
-      if ca_cert
+      if ca_cert || ca_cert_file_path
         store = OpenSSL::X509::Store.new
         Array(ca_cert).each do |cert|
           store.add_cert(OpenSSL::X509::Certificate.new(cert))
         end
+        if ca_cert_file_path
+          store.add_file(ca_cert_file_path)
+        end
         ssl_context.cert_store = store
       end
 

data/lib/kafka/connection_builder.rb

@@ -1,8 +1,9 @@
 require 'kafka/sasl_gssapi_authenticator'
+require 'kafka/sasl_plain_authenticator'
 
 module Kafka
   class ConnectionBuilder
-    def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_gssapi_principal:, sasl_gssapi_keytab:)
+    def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:, sasl_gssapi_principal:, sasl_gssapi_keytab:, sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:)
       @client_id = client_id
       @logger = logger
       @instrumenter = instrumenter
@@ -11,6 +12,9 @@ module Kafka
       @ssl_context = ssl_context
       @sasl_gssapi_principal = sasl_gssapi_principal
       @sasl_gssapi_keytab = sasl_gssapi_keytab
+      @sasl_plain_authzid = sasl_plain_authzid
+      @sasl_plain_username = sasl_plain_username
+      @sasl_plain_password = sasl_plain_password
     end
 
     def build_connection(host, port)
@@ -27,6 +31,8 @@ module Kafka
 
       if authenticate_using_sasl_gssapi?
         sasl_gssapi_authenticate(connection)
+      elsif authenticate_using_sasl_plain?
+        sasl_plain_authenticate(connection)
       end
 
       connection
@@ -45,8 +51,24 @@ module Kafka
       auth.authenticate!
     end
 
+    def sasl_plain_authenticate(connection)
+      auth = SaslPlainAuthenticator.new(
+        connection: connection,
+        logger: @logger,
+        authzid: @sasl_plain_authzid,
+        username: @sasl_plain_username,
+        password: @sasl_plain_password
+      )
+
+      auth.authenticate!
+    end
+
     def authenticate_using_sasl_gssapi?
       !@ssl_context && @sasl_gssapi_principal && !@sasl_gssapi_principal.empty?
     end
+
+    def authenticate_using_sasl_plain?
+      @sasl_plain_authzid && @sasl_plain_username && @sasl_plain_password
+    end
   end
 end

data/lib/kafka/consumer.rb

@@ -258,13 +258,13 @@ module Kafka
               begin
                 yield batch
               rescue => e
-                offset_range = batch.empty? ? "N/A" : [batch.first_offset, batch.last_offset].join("..")
+                offset_range = (batch.first_offset .. batch.last_offset)
                 location = "#{batch.topic}/#{batch.partition} in offset range #{offset_range}"
                 backtrace = e.backtrace.join("\n")
 
                 @logger.error "Exception raised when processing #{location} -- #{e.class}: #{e}\n#{backtrace}"
 
-                raise
+                raise ProcessingError.new(batch.topic, batch.partition, offset_range)
               end
             end
 
@@ -308,6 +308,9 @@ module Kafka
           @logger.error "Leader not available; waiting 1s before retrying"
           @cluster.mark_as_stale!
           sleep 1
+        rescue SignalException => e
+          @logger.warn "Received signal #{e.message}, shutting down"
+          @running = false
         end
       end
     ensure
@@ -320,7 +323,7 @@ module Kafka
 
     def make_final_offsets_commit!(attempts = 3)
       @offset_manager.commit_offsets
-    rescue ConnectionError
+    rescue ConnectionError, EOFError
       # It's important to make sure final offsets commit is done
       # As otherwise messages that have been processed after last auto-commit
       # will be processed again and that may be huge amount of messages

data/lib/kafka/protocol/sasl_handshake_request.rb

@@ -6,7 +6,7 @@ module Kafka
 
     class SaslHandshakeRequest
 
-      SUPPORTED_MECHANISMS = %w(GSSAPI)
+      SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN)
 
       def initialize(mechanism)
         unless SUPPORTED_MECHANISMS.include?(mechanism)

data/lib/kafka/sasl_plain_authenticator.rb

@@ -0,0 +1,37 @@
+module Kafka
+  class SaslPlainAuthenticator
+    PLAIN_IDENT = "PLAIN"
+
+    def initialize(connection:, logger:, authzid:, username:, password:)
+      @connection = connection
+      @logger = logger
+      @authzid = authzid
+      @username = username
+      @password = password
+    end
+
+    def authenticate!
+      response = @connection.send_request(Kafka::Protocol::SaslHandshakeRequest.new(PLAIN_IDENT))
+
+      @encoder = @connection.encoder
+      @decoder = @connection.decoder
+
+      unless response.error_code == 0 && response.enabled_mechanisms.include?(PLAIN_IDENT)
+        raise Kafka::Error, "#{PLAIN_IDENT} is not supported."
+      end
+
+      # SASL PLAIN
+      msg = [@authzid.to_s,
+             @username.to_s,
+             @password.to_s].join("\000").force_encoding('utf-8')
+      @encoder.write_bytes(msg)
+      begin
+        msg = @decoder.bytes
+        raise Kafka::Error, 'SASL PLAIN authentication failed: unknown error' unless msg
+      rescue Errno::ETIMEDOUT, EOFError => e
+        raise Kafka::Error, "SASL PLAIN authentication failed: #{e.message}"
+      end
+      @logger.debug 'SASL PLAIN authentication successful.'
+    end
+  end
+end

data/lib/kafka/statsd.rb

@@ -0,0 +1,209 @@
+begin
+  require "statsd"
+rescue LoadError
+  $stderr.puts "In order to report Kafka client metrics to Statsd you need to install the `statsd-ruby` gem."
+  raise
+end
+
+require "active_support/subscriber"
+
+module Kafka
+  # Reports operational metrics to a Statsd agent.
+  #
+  #     require "kafka/statds"
+  #
+  #     # Default is "ruby_kafka".
+  #     Kafka::Statsd.namespace = "custom-namespace"
+  #
+  #     # Default is "127.0.0.1".
+  #     Kafka::Statsd.host = "statsd.something.com"
+  #
+  #     # Default is 8125.
+  #     Kafka::Statsd.port = 1234
+  #
+  # Once the file has been required, no further configuration is needed – all operational
+  # metrics are automatically emitted.
+  module Statsd
+    DEFAULT_NAMESPACE = "ruby_kafka"
+    DEFAULT_HOST = '127.0.0.1'
+    DEFAULT_PORT = 8125
+
+    def self.statsd
+      @statsd ||= ::Statsd.new(DEFAULT_HOST, DEFAULT_PORT).tap{ |sd| sd.namespace = DEFAULT_NAMESPACE }
+    end
+
+    def self.host=(host)
+      statsd.host = host
+      statsd.connect if statsd.respond_to?(:connect)
+    end
+
+    def self.port=(port)
+      statsd.port = port
+      statsd.connect if statsd.respond_to?(:connect)
+    end
+
+    def self.namespace=(namespace)
+      statsd.namespace = namespace
+    end
+
+    class StatsdSubscriber < ActiveSupport::Subscriber
+      private
+
+      %w[increment count timing gauge].each do |type|
+        define_method(type) do |*args|
+          Kafka::Statsd.statsd.send(type, *args)
+        end
+      end
+    end
+
+    class ConnectionSubscriber < StatsdSubscriber
+      def request(event)
+        client = event.payload.fetch(:client_id)
+        api = event.payload.fetch(:api, "unknown")
+        request_size = event.payload.fetch(:request_size, 0)
+        response_size = event.payload.fetch(:response_size, 0)
+        broker = event.payload.fetch(:broker_host)
+
+        timing("api.#{client}.#{api}.#{broker}.latency", event.duration)
+        increment("api.#{client}.#{api}.#{broker}.calls")
+
+        timing("api.#{client}.#{api}.#{broker}.request_size", request_size)
+        timing("api.#{client}.#{api}.#{broker}.response_size", response_size)
+
+        if event.payload.key?(:exception)
+          increment("api.#{client}.#{api}.#{broker}.errors")
+        end
+      end
+
+      attach_to "connection.kafka"
+    end
+
+    class ConsumerSubscriber < StatsdSubscriber
+      def process_message(event)
+        lag = event.payload.fetch(:offset_lag)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+        topic = event.payload.fetch(:topic)
+        partition = event.payload.fetch(:partition)
+
+        if event.payload.key?(:exception)
+          increment("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_message.errors")
+        else
+          timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_message.latency", event.duration)
+          increment("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages")
+        end
+
+        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
+      end
+
+      def process_batch(event)
+        messages = event.payload.fetch(:message_count)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+        topic = event.payload.fetch(:topic)
+        partition = event.payload.fetch(:partition)
+
+        if event.payload.key?(:exception)
+          increment("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.errors")
+        else
+          timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
+          count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
+        end
+      end
+
+      attach_to "consumer.kafka"
+    end
+
+    class ProducerSubscriber < StatsdSubscriber
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+        max_buffer_size = event.payload.fetch(:max_buffer_size)
+        buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+
+        # This gets us the write rate.
+        increment("producer.#{client}.#{topic}.produce.messages")
+
+        timing("producer.#{client}.#{topic}.produce.message_size", message_size)
+
+        # This gets us the avg/max buffer size per producer.
+        timing("producer.#{client}.buffer.size", buffer_size)
+
+        # This gets us the avg/max buffer fill ratio per producer.
+        timing("producer.#{client}.buffer.fill_ratio", buffer_fill_ratio)
+      end
+
+      def buffer_overflow(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+
+        increment("producer.#{client}.#{topic}.produce.errors")
+      end
+
+      def deliver_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:delivered_message_count)
+        attempts = event.payload.fetch(:attempts)
+
+        if event.payload.key?(:exception)
+          increment("producer.#{client}.deliver.errors")
+        end
+
+        timing("producer.#{client}.deliver.latency", event.duration)
+
+        # Messages delivered to Kafka:
+        count("producer.#{client}.deliver.messages", message_count)
+
+        # Number of attempts to deliver messages:
+        timing("producer.#{client}.deliver.attempts", attempts)
+      end
+
+      def ack_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+
+        # Number of messages ACK'd for the topic.
+        increment("producer.#{client}.#{topic}.ack.messages")
+
+        # Histogram of delay between a message being produced and it being ACK'd.
+        timing("producer.#{client}.#{topic}.ack.delay", event.payload.fetch(:delay))
+      end
+
+      def topic_error(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+
+        increment("producer.#{client}.#{topic}.ack.errors")
+      end
+
+      attach_to "producer.kafka"
+    end
+
+    class AsyncProducerSubscriber < StatsdSubscriber
+      def enqueue_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        queue_size = event.payload.fetch(:queue_size)
+        max_queue_size = event.payload.fetch(:max_queue_size)
+        queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+        # This gets us the avg/max queue size per producer.
+        timing("async_producer.#{client}.#{topic}.queue.size", queue_size)
+
+        # This gets us the avg/max queue fill ratio per producer.
+        timing("async_producer.#{client}.#{topic}.queue.fill_ratio", queue_fill_ratio)
+      end
+
+      def buffer_overflow(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+
+        increment("async_producer.#{client}.#{topic}.produce.errors")
+      end
+
+      attach_to "async_producer.kafka"
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.18.beta2"
+  VERSION = "0.4.0.beta1"
 end

data/ruby-kafka.gemspec

@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
   spec.add_development_dependency "dogstatsd-ruby", ">= 2.0.0"
+  spec.add_development_dependency "statsd-ruby"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.18.beta2
+  version: 0.4.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-29 00:00:00.000000000 Z
+date: 2017-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gssapi
@@ -192,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: statsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -309,9 +323,11 @@ files:
 - lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/round_robin_assignment_strategy.rb
 - lib/kafka/sasl_gssapi_authenticator.rb
+- lib/kafka/sasl_plain_authenticator.rb
 - lib/kafka/snappy_codec.rb
 - lib/kafka/socket_with_timeout.rb
 - lib/kafka/ssl_socket_with_timeout.rb
+- lib/kafka/statsd.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
 - performance/profile.rb