ruby-kafka 0.7.10 → 1.5.0

data/lib/kafka/sasl/awsmskiam.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'base64'
+require 'json'
+
+module Kafka
+  module Sasl
+    class AwsMskIam
+      AWS_MSK_IAM = "AWS_MSK_IAM"
+
+      def initialize(aws_region:, access_key_id:, secret_key_id:, session_token: nil, logger:)
+        @semaphore = Mutex.new
+
+        @aws_region = aws_region
+        @access_key_id = access_key_id
+        @secret_key_id = secret_key_id
+        @session_token = session_token
+        @logger = TaggedLogger.new(logger)
+      end
+
+      def ident
+        AWS_MSK_IAM
+      end
+
+      def configured?
+        @aws_region && @access_key_id && @secret_key_id
+      end
+
+      def authenticate!(host, encoder, decoder)
+        @logger.debug "Authenticating #{@access_key_id} with SASL #{AWS_MSK_IAM}"
+
+        host_without_port = host.split(':', -1).first
+
+        time_now = Time.now.utc
+
+        msg = authentication_payload(host: host_without_port, time_now: time_now)
+        @logger.debug "Sending first client SASL AWS_MSK_IAM message:"
+        @logger.debug msg
+        encoder.write_bytes(msg)
+
+        begin
+          @server_first_message = decoder.bytes
+          @logger.debug "Received first server SASL AWS_MSK_IAM message: #{@server_first_message}"
+
+          raise Kafka::Error, "SASL AWS_MSK_IAM authentication failed: unknown error" unless @server_first_message
+        rescue Errno::ETIMEDOUT, EOFError => e
+          raise Kafka::Error, "SASL AWS_MSK_IAM authentication failed: #{e.message}"
+        end
+
+        @logger.debug "SASL #{AWS_MSK_IAM} authentication successful"
+      end
+
+      private
+
+      def bin_to_hex(s)
+        s.each_byte.map { |b| b.to_s(16).rjust(2, '0') }.join
+      end
+
+      def digest
+        @digest ||= OpenSSL::Digest::SHA256.new
+      end
+
+      def authentication_payload(host:, time_now:)
+        {
+          'version' => "2020_10_22",
+          'host' => host,
+          'user-agent' => "ruby-kafka",
+          'action' => "kafka-cluster:Connect",
+          'x-amz-algorithm' => "AWS4-HMAC-SHA256",
+          'x-amz-credential' => @access_key_id + "/" + time_now.strftime("%Y%m%d") + "/" + @aws_region + "/kafka-cluster/aws4_request",
+          'x-amz-date' => time_now.strftime("%Y%m%dT%H%M%SZ"),
+          'x-amz-signedheaders' => "host",
+          'x-amz-expires' => "900",
+          'x-amz-security-token' => @session_token,
+          'x-amz-signature' => signature(host: host, time_now: time_now)
+        }.delete_if { |_, v| v.nil? }.to_json
+      end
+
+      def canonical_request(host:, time_now:)
+        "GET\n" +
+          "/\n" +
+          canonical_query_string(time_now: time_now) + "\n" +
+          canonical_headers(host: host) + "\n" +
+          signed_headers + "\n" +
+          hashed_payload
+      end
+
+      def canonical_query_string(time_now:)
+        params = {
+          "Action" => "kafka-cluster:Connect",
+          "X-Amz-Algorithm" => "AWS4-HMAC-SHA256",
+          "X-Amz-Credential" => @access_key_id + "/" + time_now.strftime("%Y%m%d") + "/" + @aws_region + "/kafka-cluster/aws4_request",
+          "X-Amz-Date" => time_now.strftime("%Y%m%dT%H%M%SZ"),
+          "X-Amz-Expires" => "900",
+          "X-Amz-Security-Token" => @session_token,
+          "X-Amz-SignedHeaders" => "host"
+        }.delete_if { |_, v| v.nil? }
+
+        URI.encode_www_form(params)
+      end
+
+      def canonical_headers(host:)
+        "host" + ":" + host + "\n"
+      end
+
+      def signed_headers
+        "host"
+      end
+
+      def hashed_payload
+        bin_to_hex(digest.digest(""))
+      end
+
+      def string_to_sign(host:, time_now:)
+        "AWS4-HMAC-SHA256" + "\n" +
+          time_now.strftime("%Y%m%dT%H%M%SZ") + "\n" +
+          time_now.strftime("%Y%m%d") + "/" + @aws_region + "/kafka-cluster/aws4_request" + "\n" +
+          bin_to_hex(digest.digest(canonical_request(host: host, time_now: time_now)))
+      end
+
+      def signature(host:, time_now:)
+        date_key = OpenSSL::HMAC.digest("SHA256", "AWS4" + @secret_key_id, time_now.strftime("%Y%m%d"))
+        date_region_key = OpenSSL::HMAC.digest("SHA256", date_key, @aws_region)
+        date_region_service_key = OpenSSL::HMAC.digest("SHA256", date_region_key, "kafka-cluster")
+        signing_key = OpenSSL::HMAC.digest("SHA256", date_region_service_key, "aws4_request")
+        signature = bin_to_hex(OpenSSL::HMAC.digest("SHA256", signing_key, string_to_sign(host: host, time_now: time_now)))
+
+        signature
+      end
+    end
+  end
+end

data/lib/kafka/sasl_authenticator.rb

@@ -4,13 +4,18 @@ require 'kafka/sasl/plain'
 require 'kafka/sasl/gssapi'
 require 'kafka/sasl/scram'
 require 'kafka/sasl/oauth'
+require 'kafka/sasl/awsmskiam'
 
 module Kafka
   class SaslAuthenticator
     def initialize(logger:, sasl_gssapi_principal:, sasl_gssapi_keytab:,
                    sasl_plain_authzid:, sasl_plain_username:, sasl_plain_password:,
                    sasl_scram_username:, sasl_scram_password:, sasl_scram_mechanism:,
-                   sasl_oauth_token_provider:)
+                   sasl_oauth_token_provider:,
+                   sasl_aws_msk_iam_access_key_id:,
+                   sasl_aws_msk_iam_secret_key_id:,
+                   sasl_aws_msk_iam_aws_region:,
+                   sasl_aws_msk_iam_session_token: nil)
       @logger = TaggedLogger.new(logger)
 
       @plain = Sasl::Plain.new(
@@ -33,12 +38,20 @@ module Kafka
         logger: @logger,
       )
 
+      @aws_msk_iam = Sasl::AwsMskIam.new(
+        access_key_id: sasl_aws_msk_iam_access_key_id,
+        secret_key_id: sasl_aws_msk_iam_secret_key_id,
+        aws_region: sasl_aws_msk_iam_aws_region,
+        session_token: sasl_aws_msk_iam_session_token,
+        logger: @logger,
+      )
+
       @oauth = Sasl::OAuth.new(
         token_provider: sasl_oauth_token_provider,
         logger: @logger,
       )
 
-      @mechanism = [@gssapi, @plain, @scram, @oauth].find(&:configured?)
+      @mechanism = [@gssapi, @plain, @scram, @oauth, @aws_msk_iam].find(&:configured?)
     end
 
     def enabled?

data/lib/kafka/ssl_context.rb

@@ -47,18 +47,19 @@ module Kafka
         Array(ca_cert).each do |cert|
           store.add_cert(OpenSSL::X509::Certificate.new(cert))
         end
-        if ca_cert_file_path
-          store.add_file(ca_cert_file_path)
+        Array(ca_cert_file_path).each do |cert_file_path|
+          store.add_file(cert_file_path)
         end
         if ca_certs_from_system
           store.set_default_paths
         end
         ssl_context.cert_store = store
-        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        # Verify certificate hostname if supported (ruby >= 2.4.0)
-        ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
       end
 
+      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # Verify certificate hostname if supported (ruby >= 2.4.0)
+      ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
+
       ssl_context
     end
   end

data/lib/kafka/tagged_logger.rb

@@ -1,6 +1,7 @@
 # Basic implementation of a tagged logger that matches the API of
 # ActiveSupport::TaggedLogging.
 
+require 'delegate'
 require 'logger'
 
 module Kafka

data/lib/kafka/transaction_manager.rb

@@ -95,7 +95,7 @@ module Kafka
       force_transactional!
 
       if @transaction_state.uninitialized?
-        raise 'Transaction is uninitialized'
+        raise Kafka::InvalidTxnStateError, 'Transaction is uninitialized'
       end
 
       # Extract newly created partitions
@@ -138,8 +138,8 @@ module Kafka
 
     def begin_transaction
       force_transactional!
-      raise 'Transaction has already started' if @transaction_state.in_transaction?
-      raise 'Transaction is not ready' unless @transaction_state.ready?
+      raise Kafka::InvalidTxnStateError, 'Transaction has already started' if @transaction_state.in_transaction?
+      raise Kafka::InvalidTxnStateError, 'Transaction is not ready' unless @transaction_state.ready?
       @transaction_state.transition_to!(TransactionStateMachine::IN_TRANSACTION)
 
       @logger.info "Begin transaction #{@transactional_id}, Producer ID: #{@producer_id} (Epoch #{@producer_epoch})"
@@ -159,7 +159,7 @@ module Kafka
       end
 
       unless @transaction_state.in_transaction?
-        raise 'Transaction is not valid to commit'
+        raise Kafka::InvalidTxnStateError, 'Transaction is not valid to commit'
       end
 
       @transaction_state.transition_to!(TransactionStateMachine::COMMITTING_TRANSACTION)
@@ -192,7 +192,8 @@ module Kafka
       end
 
       unless @transaction_state.in_transaction?
-        raise 'Transaction is not valid to abort'
+        @logger.warn('Aborting transaction that was never opened on brokers')
+        return
       end
 
       @transaction_state.transition_to!(TransactionStateMachine::ABORTING_TRANSACTION)
@@ -221,7 +222,7 @@ module Kafka
       force_transactional!
 
       unless @transaction_state.in_transaction?
-        raise 'Transaction is not valid to send offsets'
+        raise Kafka::InvalidTxnStateError, 'Transaction is not valid to send offsets'
       end
 
       add_response = transaction_coordinator.add_offsets_to_txn(
@@ -232,14 +233,23 @@ module Kafka
       )
       Protocol.handle_error(add_response.error_code)
 
-      send_response = transaction_coordinator.txn_offset_commit(
+      send_response = group_coordinator(group_id: group_id).txn_offset_commit(
         transactional_id: @transactional_id,
         group_id: group_id,
         producer_id: @producer_id,
         producer_epoch: @producer_epoch,
         offsets: offsets
       )
-      Protocol.handle_error(send_response.error_code)
+      send_response.errors.each do |tp|
+        tp.partitions.each do |partition|
+          Protocol.handle_error(partition.error_code)
+        end
+      end
+
+      nil
+    rescue
+      @transaction_state.transition_to!(TransactionStateMachine::ERROR)
+      raise
     end
 
     def in_transaction?
@@ -250,6 +260,10 @@ module Kafka
       @transaction_state.error?
     end
 
+    def ready?
+      @transaction_state.ready?
+    end
+
     def close
       if in_transaction?
         @logger.warn("Aborting pending transaction ...")
@@ -264,11 +278,11 @@ module Kafka
 
     def force_transactional!
       unless transactional?
-        raise 'Please turn on transactional mode to use transaction'
+        raise Kafka::InvalidTxnStateError, 'Please turn on transactional mode to use transaction'
       end
 
       if @transactional_id.nil? || @transactional_id.empty?
-        raise 'Please provide a transaction_id to use transactional mode'
+        raise Kafka::InvalidTxnStateError, 'Please provide a transaction_id to use transactional mode'
       end
     end
 
@@ -278,6 +292,12 @@ module Kafka
       )
     end
 
+    def group_coordinator(group_id:)
+      @cluster.get_group_coordinator(
+        group_id: group_id
+      )
+    end
+
     def complete_transaction
       @transaction_state.transition_to!(TransactionStateMachine::READY)
       @transaction_partitions = {}

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.10"
+  VERSION = "1.5.0"
 end

data/ruby-kafka.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
-  spec.license       = "Apache License Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.required_ruby_version = '>= 2.1.0'
 
@@ -33,18 +33,19 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "digest-murmurhash"
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "extlz4"
   spec.add_development_dependency "zstd-ruby"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
   spec.add_development_dependency "statsd-ruby"
-  spec.add_development_dependency "prometheus-client"
+  spec.add_development_dependency "prometheus-client", "~> 0.10.0"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.10
+  version: 1.5.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-07 00:00:00.000000000 Z
+date: 2022-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: digest-murmurhash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +142,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: snappy
   requirement: !ruby/object:Gem::Requirement
@@ -212,7 +232,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -222,7 +242,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -244,16 +264,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +352,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".readygo"
 - ".rspec"
@@ -369,7 +390,10 @@ files:
 - lib/kafka/connection_builder.rb
 - lib/kafka/consumer.rb
 - lib/kafka/consumer_group.rb
+- lib/kafka/consumer_group/assignor.rb
+- lib/kafka/crc32_hash.rb
 - lib/kafka/datadog.rb
+- lib/kafka/digest.rb
 - lib/kafka/fetch_operation.rb
 - lib/kafka/fetched_batch.rb
 - lib/kafka/fetched_batch_generator.rb
@@ -379,8 +403,10 @@ files:
 - lib/kafka/gzip_codec.rb
 - lib/kafka/heartbeat.rb
 - lib/kafka/instrumenter.rb
+- lib/kafka/interceptors.rb
 - lib/kafka/lz4_codec.rb
 - lib/kafka/message_buffer.rb
+- lib/kafka/murmur2_hash.rb
 - lib/kafka/offset_manager.rb
 - lib/kafka/partitioner.rb
 - lib/kafka/pause.rb
@@ -450,6 +476,7 @@ files:
 - lib/kafka/protocol/txn_offset_commit_request.rb
 - lib/kafka/protocol/txn_offset_commit_response.rb
 - lib/kafka/round_robin_assignment_strategy.rb
+- lib/kafka/sasl/awsmskiam.rb
 - lib/kafka/sasl/gssapi.rb
 - lib/kafka/sasl/oauth.rb
 - lib/kafka/sasl/plain.rb
@@ -469,7 +496,7 @@ files:
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
-- Apache License Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -486,8 +513,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A client library for the Kafka distributed commit log.