ruby-kafka 0.7.0 → 0.7.1.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d27b2212172f3f44a3402ad32c7b30413d8e3b0eef28e9d46161c977740fb27
-  data.tar.gz: 58345eea3931d45849a46913d03e29887e2257bffafbb9a1ca531b700ed6960f
+  metadata.gz: c6a5d1dd1969107f9107675cad6848630f8cf13ce4ce223b0fa7c84bc4facf03
+  data.tar.gz: 2797042cbef4be8b219d2103dc5f63cbe36bb5c68aae738178ed02b197579254
 SHA512:
-  metadata.gz: 1b64e322958a28c917efcd9a66f1bf3f2a1b17403830b915467670a0cedad9a810eed2d472c5359c8af963c4d890f0ca8d20175831cb628a9406548b23834607
-  data.tar.gz: 5dfb9df17ca7cc57b6435f03cf690e39517132ac3170c78e090781af20f72b91893785b03f864c9e460e2f9f1fca93149bd916975351c38bbd32606a6ce1b323
+  metadata.gz: d3dbfbb842fbc7b370d197a6f8020d4c88814c64383fe9cacd8e4338e4cafbdf222573476ba6bcf4d4ab05263b30ab0b63de2b7bc861d06730a303aec0a8d7a9
+  data.tar.gz: 12aba0aeacb8ceb731591f037ef075b3218d6c44710c5852fdfa1e5c769649bea95b4d609a9920ba48ce80b207a82628c488759d4c7cb48317ce71bbee36bfd0

data/.circleci/config.yml

@@ -41,7 +41,7 @@ jobs:
       - run: bundle install --path vendor/bundle
       - run: bundle exec rspec --profile --tag functional spec/functional
 
-  kafka-1.0:
+  kafka-1.0.0:
     docker:
       - image: circleci/ruby:2.5.1-node
         environment:
@@ -111,5 +111,5 @@ workflows:
     jobs:
       - unit
       - kafka-0.11
-      - kafka-1.0
-      - kafka-1.1
+      - kafka-1.0.0
+      - kafka-1.1

data/.gitignore

@@ -10,3 +10,4 @@
 .env
 *.log
 *.swp
+.byebug_history

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## 0.7.1
+
+- Exactly Once Delivery and Transactional Messaging Support (#608).
+
 ## 0.7.0
 
 - Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.

data/lib/kafka.rb

@@ -245,6 +245,38 @@ module Kafka
   class FailedScramAuthentication < SaslScramError
   end
 
+  # The broker received an out of order sequence number
+  class OutOfOrderSequenceNumberError < Error
+  end
+
+  # The broker received a duplicate sequence number
+  class DuplicateSequenceNumberError < Error
+  end
+
+  # Producer attempted an operation with an old epoch. Either there is a newer producer with the same transactionalId, or the producer's transaction has been expired by the broker.
+  class InvalidProducerEpochError < Error
+  end
+
+  # The producer attempted a transactional operation in an invalid state
+  class InvalidTxnStateError < Error
+  end
+
+  # The producer attempted to use a producer id which is not currently assigned to its transactional id
+  class InvalidProducerIDMappingError < Error
+  end
+
+  # The transaction timeout is larger than the maximum value allowed by the broker (as configured by transaction.max.timeout.ms).
+  class InvalidTransactionTimeoutError < Error
+  end
+
+  # The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing
+  class ConcurrentTransactionError < Error
+  end
+
+  # Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer
+  class TransactionCoordinatorFencedError < Error
+  end
+
   # Initializes a new Kafka client.
   #
   # @see Client#initialize

data/lib/kafka/broker.rb

@@ -164,6 +164,24 @@ module Kafka
       send_request(request)
     end
 
+    def init_producer_id(**options)
+      request = Protocol::InitProducerIDRequest.new(**options)
+
+      send_request(request)
+    end
+
+    def add_partitions_to_txn(**options)
+      request = Protocol::AddPartitionsToTxnRequest.new(**options)
+
+      send_request(request)
+    end
+
+    def end_txn(**options)
+      request = Protocol::EndTxnRequest.new(**options)
+
+      send_request(request)
+    end
+
     private
 
     def send_request(request)

data/lib/kafka/client.rb

@@ -2,6 +2,7 @@
 
 require "kafka/ssl_context"
 require "kafka/cluster"
+require "kafka/transaction_manager"
 require "kafka/broker_info"
 require "kafka/producer"
 require "kafka/consumer"
@@ -60,7 +61,7 @@ module Kafka
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil,
                    ssl_ca_cert_file_path: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
-                   sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
+                   ssl_client_cert_chain: nil, sasl_gssapi_principal: nil, sasl_gssapi_keytab: nil,
                    sasl_plain_authzid: '', sasl_plain_username: nil, sasl_plain_password: nil,
                    sasl_scram_username: nil, sasl_scram_password: nil, sasl_scram_mechanism: nil,
                    sasl_over_ssl: true, ssl_ca_certs_from_system: false)
@@ -73,6 +74,7 @@ module Kafka
         ca_cert: ssl_ca_cert,
         client_cert: ssl_client_cert,
         client_cert_key: ssl_client_cert_key,
+        client_cert_chain: ssl_client_cert_chain,
         ca_certs_from_system: ssl_ca_certs_from_system,
       )
 
@@ -158,8 +160,16 @@ module Kafka
         instrumenter: @instrumenter,
       )
 
+      transaction_manager = TransactionManager.new(
+        cluster: @cluster,
+        logger: @logger,
+        idempotent: false,
+        transactional: false
+      )
+
       operation = ProduceOperation.new(
         cluster: @cluster,
+        transaction_manager: transaction_manager,
         buffer: buffer,
         required_acks: 1,
         ack_timeout: 10,
@@ -222,15 +232,39 @@ module Kafka
     #   are per-partition rather than per-topic or per-producer.
     #
     # @return [Kafka::Producer] the Kafka producer.
-    def producer(compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
+    def producer(
+      compression_codec: nil,
+      compression_threshold: 1,
+      ack_timeout: 5,
+      required_acks: :all,
+      max_retries: 2,
+      retry_backoff: 1,
+      max_buffer_size: 1000,
+      max_buffer_bytesize: 10_000_000,
+      idempotent: false,
+      transactional: false,
+      transactional_id: nil,
+      transactional_timeout: 60
+    )
+      cluster = initialize_cluster
       compressor = Compressor.new(
         codec_name: compression_codec,
         threshold: compression_threshold,
         instrumenter: @instrumenter,
       )
 
+      transaction_manager = TransactionManager.new(
+        cluster: cluster,
+        logger: @logger,
+        idempotent: idempotent,
+        transactional: transactional,
+        transactional_id: transactional_id,
+        transactional_timeout: transactional_timeout,
+      )
+
       Producer.new(
-        cluster: initialize_cluster,
+        cluster: cluster,
+        transaction_manager: transaction_manager,
         logger: @logger,
         instrumenter: @instrumenter,
         compressor: compressor,
@@ -478,7 +512,7 @@ module Kafka
 
         batches.each do |batch|
           batch.messages.each(&block)
-          offsets[batch.partition] = batch.last_offset + 1 unless batch.empty?
+          offsets[batch.partition] = batch.last_offset + 1 unless batch.unknown_last_offset?
         end
       end
     end

data/lib/kafka/cluster.rb

@@ -111,48 +111,32 @@ module Kafka
       connect_to_broker(get_leader_id(topic, partition))
     end
 
+    # Finds the broker acting as the coordinator of the given group.
+    #
+    # @param group_id: [String]
+    # @return [Broker] the broker that's currently coordinator.
     def get_group_coordinator(group_id:)
       @logger.debug "Getting group coordinator for `#{group_id}`"
-
       refresh_metadata_if_necessary!
+      get_coordinator(Kafka::Protocol::COORDINATOR_TYPE_GROUP, group_id)
+    end
 
-      cluster_info.brokers.each do |broker_info|
-        begin
-          broker = connect_to_broker(broker_info.node_id)
-          response = broker.find_coordinator(
-            coordinator_type: Kafka::Protocol::COORDINATOR_TYPE_GROUP,
-            coordinator_key: group_id
-          )
-
-          Protocol.handle_error(response.error_code, response.error_message)
-
-          coordinator_id = response.coordinator_id
-
-          @logger.debug "Coordinator for group `#{group_id}` is #{coordinator_id}. Connecting..."
-
-          # It's possible that a new broker is introduced to the cluster and
-          # becomes the coordinator before we have a chance to refresh_metadata.
-          coordinator = begin
-            connect_to_broker(coordinator_id)
-          rescue Kafka::NoSuchBroker
-            @logger.debug "Broker #{coordinator_id} missing from broker cache, refreshing"
-            refresh_metadata!
-            connect_to_broker(coordinator_id)
-          end
+    # Finds the broker acting as the coordinator of the given transaction.
+    #
+    # @param transactional_id: [String]
+    # @return [Broker] the broker that's currently coordinator.
+    def get_transaction_coordinator(transactional_id:)
+      @logger.debug "Getting transaction coordinator for `#{transactional_id}`"
 
-          @logger.debug "Connected to coordinator: #{coordinator} for group `#{group_id}`"
+      refresh_metadata_if_necessary!
 
-          return coordinator
-        rescue CoordinatorNotAvailable
-          @logger.debug "Coordinator not available; retrying in 1s"
-          sleep 1
-          retry
-        rescue ConnectionError => e
-          @logger.error "Failed to get group coordinator info from #{broker}: #{e}"
-        end
+      if transactional_id.nil?
+        # Get a random_broker
+        @logger.debug "Transaction ID is not available. Choose a random broker."
+        return random_broker
+      else
+        get_coordinator(Kafka::Protocol::COORDINATOR_TYPE_TRANSACTION, transactional_id)
       end
-
-      raise Kafka::Error, "Failed to find group coordinator"
     end
 
     def partitions_for(topic)
@@ -313,8 +297,7 @@ module Kafka
             topic => broker_partitions.map {|partition|
               {
                 partition: partition,
-                time: offset,
-                max_offsets: 1,
+                time: offset
               }
             }
           }
@@ -427,5 +410,45 @@ module Kafka
     def controller_broker
       connect_to_broker(cluster_info.controller_id)
     end
+
+    def get_coordinator(coordinator_type, coordinator_key)
+      cluster_info.brokers.each do |broker_info|
+        begin
+          broker = connect_to_broker(broker_info.node_id)
+          response = broker.find_coordinator(
+            coordinator_type: coordinator_type,
+            coordinator_key: coordinator_key
+          )
+
+          Protocol.handle_error(response.error_code, response.error_message)
+
+          coordinator_id = response.coordinator_id
+
+          @logger.debug "Coordinator for `#{coordinator_key}` is #{coordinator_id}. Connecting..."
+
+          # It's possible that a new broker is introduced to the cluster and
+          # becomes the coordinator before we have a chance to refresh_metadata.
+          coordinator = begin
+            connect_to_broker(coordinator_id)
+          rescue Kafka::NoSuchBroker
+            @logger.debug "Broker #{coordinator_id} missing from broker cache, refreshing"
+            refresh_metadata!
+            connect_to_broker(coordinator_id)
+          end
+
+          @logger.debug "Connected to coordinator: #{coordinator} for `#{coordinator_key}`"
+
+          return coordinator
+        rescue CoordinatorNotAvailable
+          @logger.debug "Coordinator not available; retrying in 1s"
+          sleep 1
+          retry
+        rescue ConnectionError => e
+          @logger.error "Failed to get coordinator info from #{broker}: #{e}"
+        end
+      end
+
+      raise Kafka::Error, "Failed to find coordinator"
+    end
   end
 end

data/lib/kafka/consumer.rb

@@ -310,9 +310,9 @@ module Kafka
             @instrumenter.instrument("process_batch.consumer", notification) do
               begin
                 yield batch
-                @current_offsets[batch.topic][batch.partition] = batch.last_offset
+                @current_offsets[batch.topic][batch.partition] = batch.last_offset unless batch.unknown_last_offset?
               rescue => e
-                offset_range = (batch.first_offset..batch.last_offset)
+                offset_range = (batch.first_offset..batch.last_offset || batch.highwater_mark_offset)
                 location = "#{batch.topic}/#{batch.partition} in offset range #{offset_range}"
                 backtrace = e.backtrace.join("\n")
 

data/lib/kafka/fetch_operation.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require "kafka/fetched_batch"
+require "kafka/fetched_offset_resolver"
+require "kafka/fetched_batch_generator"
 
 module Kafka
 
@@ -27,6 +28,10 @@ module Kafka
       @max_bytes = max_bytes
       @max_wait_time = max_wait_time
       @topics = {}
+
+      @offset_resolver = Kafka::FetchedOffsetResolver.new(
+        logger: logger
+      )
     end
 
     def fetch_from_partition(topic, partition, offset: :latest, max_bytes: 1048576)
@@ -63,8 +68,8 @@ module Kafka
         end
       end
 
-      topics_by_broker.flat_map {|broker, topics|
-        resolve_offsets(broker, topics)
+      topics_by_broker.flat_map do |broker, topics|
+        @offset_resolver.resolve!(broker, topics)
 
         options = {
           max_wait_time: @max_wait_time * 1000, # Kafka expects ms, not secs
@@ -75,8 +80,8 @@ module Kafka
 
         response = broker.fetch_messages(**options)
 
-        response.topics.flat_map {|fetched_topic|
-          fetched_topic.partitions.map {|fetched_partition|
+        response.topics.flat_map do |fetched_topic|
+          fetched_topic.partitions.map do |fetched_partition|
             begin
               Protocol.handle_error(fetched_partition.error_code)
             rescue Kafka::OffsetOutOfRange => e
@@ -92,64 +97,18 @@ module Kafka
               raise e
             end
 
-            messages = fetched_partition.messages.map {|message|
-              FetchedMessage.new(
-                message: message,
-                topic: fetched_topic.name,
-                partition: fetched_partition.partition,
-              )
-            }
-
-            FetchedBatch.new(
-              topic: fetched_topic.name,
-              partition: fetched_partition.partition,
-              highwater_mark_offset: fetched_partition.highwater_mark_offset,
-              messages: messages,
-            )
-          }
-        }
-      }
+            Kafka::FetchedBatchGenerator.new(
+              fetched_topic.name,
+              fetched_partition,
+              logger: @logger
+            ).generate
+          end
+        end
+      end
     rescue Kafka::ConnectionError, Kafka::LeaderNotAvailable, Kafka::NotLeaderForPartition
       @cluster.mark_as_stale!
 
       raise
     end
-
-    private
-
-    def resolve_offsets(broker, topics)
-      pending_topics = {}
-
-      topics.each do |topic, partitions|
-        partitions.each do |partition, options|
-          offset = options.fetch(:fetch_offset)
-          next if offset >= 0
-
-          @logger.debug "Resolving offset `#{offset}` for #{topic}/#{partition}..."
-
-          pending_topics[topic] ||= []
-          pending_topics[topic] << {
-            partition: partition,
-            time: offset,
-            max_offsets: 1,
-          }
-        end
-      end
-
-      return topics if pending_topics.empty?
-
-      response = broker.list_offsets(topics: pending_topics)
-
-      pending_topics.each do |topic, partitions|
-        partitions.each do |options|
-          partition = options.fetch(:partition)
-          resolved_offset = response.offset_for(topic, partition)
-
-          @logger.debug "Offset for #{topic}/#{partition} is #{resolved_offset.inspect}"
-
-          topics[topic][partition][:fetch_offset] = resolved_offset || 0
-        end
-      end
-    end
   end
 end