ruby-kafka 0.7.6.beta1 → 0.7.6.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e372f99c56ceb1e59bf7bc9d6cdc6d002637dc41c40b7c3f1d782aeea1e29c40
-  data.tar.gz: 70ec79b24ffb7105de1d01ebc2ee43da6d13778b2813feacbc30707da072d1c0
+  metadata.gz: b14f3fd396d495fc5c240cd5451b7806154b0fda6dbae72764cfa8087a4b778d
+  data.tar.gz: e283a412d4bcdfd7b6ac8f8c0ba7d6ef9b1bbc94163796ca23cbe6d884710076
 SHA512:
-  metadata.gz: c072eb8640de239ca77e3397933ed9d6575f42621aacff709d966b4213b0dbef8b6dd198c3d12298167cd07349c8d4438e3127677803f2630c2c405c3e9771eb
-  data.tar.gz: 6b265b87a57d488d36c8b91c223ebb781a6b2c602fae101023b412736bbf9a7ccda1456d66eab107c22409fb1a3e6f7a0552dd274f595ef243b227206350a445
+  metadata.gz: 1bbe81c129b203d3a4f7f64853cdd21ab99b602844fc426726a74878b40bf9266a9ed14a77b05bab5e500118aa35ea8a0e39bdbf466018d746e744580a525c66
+  data.tar.gz: 359e818723d15663dc6d1de2b83b70aad4f27bd293b0bb95426be16a440f9c9c48cd58d53a754d8e62d541b8043ef8109ab7549173e2232732ebe94673acf301

data/CHANGELOG.md

@@ -8,6 +8,8 @@ Changes and additions to the library will be listed here.
 - Introduce regex matching in `Consumer#subscribe` (#700)
 - Only rejoin group on error if we're not in shutdown mode (#711)
 - Use `maxTimestamp` for `logAppendTime` timestamps (#706)
+- Async producer limit number of retries (#708)
+- Support SASL OAuthBearer Authentication (#710)
 
 ## 0.7.5
 - Distribute partitions across consumer groups when there are few partitions per topic (#681)

data/README.md

@@ -988,6 +988,26 @@ kafka = Kafka.new(
 )
 ```
 
+##### OAUTHBEARER
+This mechanism is supported in kafka >= 2.0.0 as of [KIP-255](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=75968876)
+
+In order to authenticate using OAUTHBEARER, you must set the client with an instance of a class that implements a `token` method (the interface is described in [Kafka::Sasl::OAuth](lib/kafka/sasl/oauth.rb)) which returns an ID/Access token.
+
+Optionally, the client may implement an `extensions` method that returns a map of key-value pairs. These can be sent with the SASL/OAUTHBEARER initial client response. This is only supported in kafka >= 2.1.0.
+
+```ruby
+class TokenProvider
+  def token
+    "some_id_token"
+  end
+end
+# ...
+client = Kafka.new(
+  ["kafka1:9092"],
+  sasl_oauth_token_provider: TokenProvider.new
+)
+```
+
 ### Topic management
 
 In addition to producing and consuming messages, ruby-kafka supports managing Kafka topics and their configurations. See [the Kafka documentation](https://kafka.apache.org/documentation/#topicconfigs) for a full list of topic configuration keys.

data/lib/kafka.rb

@@ -351,6 +351,10 @@ module Kafka
   class FailedScramAuthentication < SaslScramError
   end
 
+  # The Token Provider object used for SASL OAuthBearer does not implement the method `token`
+  class TokenMethodNotImplementedError < Error
+  end
+
   # Initializes a new Kafka client.
   #
   # @see Client#initialize

data/lib/kafka/async_producer.rb

@@ -72,7 +72,7 @@ module Kafka
     # @param delivery_interval [Integer] if greater than zero, the number of
     #   seconds between automatic message deliveries.
     #
-    def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, instrumenter:, logger:)
+    def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, max_retries: -1, retry_backoff: 0, instrumenter:, logger:)
       raise ArgumentError unless max_queue_size > 0
       raise ArgumentError unless delivery_threshold >= 0
       raise ArgumentError unless delivery_interval >= 0
@@ -86,8 +86,10 @@ module Kafka
         queue: @queue,
         producer: sync_producer,
         delivery_threshold: delivery_threshold,
+        max_retries: max_retries,
+        retry_backoff: retry_backoff,
         instrumenter: instrumenter,
-        logger: logger,
+        logger: logger
       )
 
       # The timer will no-op if the delivery interval is zero.
@@ -184,10 +186,12 @@ module Kafka
     end
 
     class Worker
-      def initialize(queue:, producer:, delivery_threshold:, instrumenter:, logger:)
+      def initialize(queue:, producer:, delivery_threshold:, max_retries: -1, retry_backoff: 0, instrumenter:, logger:)
         @queue = queue
         @producer = producer
         @delivery_threshold = delivery_threshold
+        @max_retries = max_retries
+        @retry_backoff = retry_backoff
         @instrumenter = instrumenter
         @logger = TaggedLogger.new(logger)
       end
@@ -240,10 +244,22 @@ module Kafka
       private
 
       def produce(*args)
-        @producer.produce(*args)
-      rescue BufferOverflow
-        deliver_messages
-        retry
+        retries = 0
+        begin
+          @producer.produce(*args)
+        rescue BufferOverflow => e
+          deliver_messages
+          if @max_retries == -1
+            retry
+          elsif retries < @max_retries
+            retries += 1
+            sleep @retry_backoff**retries
+            retry
+          else
+            @logger.error("Failed to asynchronously produce messages due to BufferOverflow")
+            @instrumenter.instrument("error.async_producer", { error: e })
+          end
+        end
       end
 
       def deliver_messages

data/lib/kafka/client.rb

@@ -62,13 +62,16 @@ module Kafka
     #
     # @param sasl_over_ssl [Boolean] whether to enforce SSL with SASL
     #
+    # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
+    #   implements method token. See {Sasl::OAuth#initialize}
+    #
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
                    ssl_client_cert_key_password: nil, ssl_client_cert_chain: nil, sasl_gssapi_principal: nil,
                    sasl_gssapi_keytab: nil, sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
                    sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
-                   sasl_over_ssl: true, ssl_ca_certs_from_system: false)
+                   sasl_over_ssl: true, ssl_ca_certs_from_system: false, sasl_oauth_token_provider: nil)
       @logger = TaggedLogger.new(logger)
       @instrumenter = Instrumenter.new(client_id: client_id)
       @seed_brokers = normalize_seed_brokers(seed_brokers)
@@ -92,6 +95,7 @@ module Kafka
         sasl_scram_username: sasl_scram_username,
         sasl_scram_password: sasl_scram_password,
         sasl_scram_mechanism: sasl_scram_mechanism,
+        sasl_oauth_token_provider: sasl_oauth_token_provider,
         logger: @logger
       )
 
@@ -296,7 +300,7 @@ module Kafka
     #
     # @see AsyncProducer
     # @return [AsyncProducer]
-    def async_producer(delivery_interval: 0, delivery_threshold: 0, max_queue_size: 1000, **options)
+    def async_producer(delivery_interval: 0, delivery_threshold: 0, max_queue_size: 1000, max_retries: -1, retry_backoff: 0, **options)
       sync_producer = producer(**options)
 
       AsyncProducer.new(
@@ -304,6 +308,8 @@ module Kafka
         delivery_interval: delivery_interval,
         delivery_threshold: delivery_threshold,
         max_queue_size: max_queue_size,
+        max_retries: max_retries,
+        retry_backoff: retry_backoff,
         instrumenter: @instrumenter,
         logger: @logger,
       )

data/lib/kafka/protocol/sasl_handshake_request.rb

@@ -8,7 +8,7 @@ module Kafka
 
     class SaslHandshakeRequest
 
-      SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512)
+      SUPPORTED_MECHANISMS = %w(GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512 OAUTHBEARER)
 
       def initialize(mechanism)
         unless SUPPORTED_MECHANISMS.include?(mechanism)

data/lib/kafka/sasl/oauth.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Kafka
+  module Sasl
+    class OAuth
+      OAUTH_IDENT = "OAUTHBEARER"
+
+      # token_provider: THE FOLLOWING INTERFACE MUST BE FULFILLED:
+      #
+      # [REQUIRED] TokenProvider#token      - Returns an ID/Access Token to be sent to the Kafka client.
+      #   The implementation should ensure token reuse so that multiple calls at connect time do not
+      #   create multiple tokens. The implementation should also periodically refresh the token in
+      #   order to guarantee that each call returns an unexpired token. A timeout error should
+      #   be returned after a short period of inactivity so that the broker can log debugging
+      #   info and retry.
+      #
+      # [OPTIONAL] TokenProvider#extensions - Returns a map of key-value pairs that can be sent with the
+      #   SASL/OAUTHBEARER initial client response. If not provided, the values are ignored. This feature
+      #   is only available in Kafka >= 2.1.0.
+      #
+      def initialize(logger:, token_provider:)
+        @logger = TaggedLogger.new(logger)
+        @token_provider = token_provider
+      end
+
+      def ident
+        OAUTH_IDENT
+      end
+
+      def configured?
+        @token_provider
+      end
+
+      def authenticate!(host, encoder, decoder)
+        # Send SASLOauthBearerClientResponse with token
+        @logger.debug "Authenticating to #{host} with SASL #{OAUTH_IDENT}"
+
+        encoder.write_bytes(initial_client_response)
+
+        begin
+          # receive SASL OAuthBearer Server Response
+          msg = decoder.bytes
+          raise Kafka::Error, "SASL #{OAUTH_IDENT} authentication failed: unknown error" unless msg
+        rescue Errno::ETIMEDOUT, EOFError => e
+          raise Kafka::Error, "SASL #{OAUTH_IDENT} authentication failed: #{e.message}"
+        end
+
+        @logger.debug "SASL #{OAUTH_IDENT} authentication successful."
+      end
+
+      private
+
+      def initial_client_response
+        raise Kafka::TokenMethodNotImplementedError, "Token provider doesn't define 'token'" unless @token_provider.respond_to? :token
+        "n,,\x01auth=Bearer #{@token_provider.token}#{token_extensions}\x01\x01"
+      end
+
+      def token_extensions
+        return nil unless @token_provider.respond_to? :extensions
+        "\x01#{@token_provider.extensions.map {|e| e.join("=")}.join("\x01")}"
+      end
+    end
+  end
+end

data/lib/kafka/sasl_authenticator.rb

@@ -3,12 +3,14 @@
 require 'kafka/sasl/plain'
 require 'kafka/sasl/gssapi'
 require 'kafka/sasl/scram'
+require 'kafka/sasl/oauth'
 
 module Kafka
   class SaslAuthenticator
     def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
                    sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:,
-                   sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:)
+                   sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:,
+                   sasl_oauth_token_provider:)
       @logger = TaggedLogger.new(logger)
 
       @plain = Sasl::Plain.new(
@@ -31,7 +33,12 @@ module Kafka
         logger: @logger,
       )
 
-      @mechanism = [@gssapi, @plain, @scram].find(&:configured?)
+      @oauth = Sasl::OAuth.new(
+        token_provider: sasl_oauth_token_provider,
+        logger: @logger,
+      )
+
+      @mechanism = [@gssapi, @plain, @scram, @oauth].find(&:configured?)
     end
 
     def enabled?

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.6.beta1"
+  VERSION = "0.7.6.beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.6.beta1
+  version: 0.7.6.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-20 00:00:00.000000000 Z
+date: 2019-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -418,6 +418,7 @@ files:
 - lib/kafka/protocol/sync_group_response.rb
 - lib/kafka/round_robin_assignment_strategy.rb
 - lib/kafka/sasl/gssapi.rb
+- lib/kafka/sasl/oauth.rb
 - lib/kafka/sasl/plain.rb
 - lib/kafka/sasl/scram.rb
 - lib/kafka/sasl_authenticator.rb