ruby-kafka 0.5.5 → 0.6.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63ec54e4024c83bb5a73583700391db7b89fed005bb4b196cf912424351bba0e
-  data.tar.gz: d567d51c5434f6ef034b769f78cc03286d91e875aa5ea9058623cd2e755cf28c
+  metadata.gz: cc9b975d79ef5be40b82d8f534995aa397d88a12b557f0b436bea4e42c0d7baa
+  data.tar.gz: f2e1fa46cbd12f0bc551f527aabd0469d5dd0bcc89bdcd27a71e05dedf9bd474
 SHA512:
-  metadata.gz: 3f9f8269075c30eb60a3ee7afcca5c1fe8eab9aa6c5cf51c5512dc6b564299a5366d33bc97d08b61a17ecc4a299e066ecb48f86938c93643324a8083289ed3af
-  data.tar.gz: 92767b83a152bef74992d1b4dc4b1818bdce8715a4e1e32c53671eaf6ee8d28d8e76d4800bf141c8cdd6cc44ef51aef78335464dbe504f8f72245ec8fc964a9a
+  metadata.gz: 60d506503b7bb77cde82b6a606bbe73ea5f8da247b4acd676a933ca91daa8e421d8beb91642469b4847725a0b6c4cfa191c74fc1f12413846b5e4bf5dd14186c
+  data.tar.gz: 9b9a7733579b0ed9c81546ea6f451df85fdbf88c9aa9ab040d81ab49f36c3d029902ae69815d07f2118a93bdf410429cd019a96d90b79b0df47eb8cbc15c2daa

data/CHANGELOG.md

@@ -4,6 +4,8 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+- Fetch messages asynchronously (#526).
+
 ## v0.5.5
 
 - Support PLAINTEXT and SSL URI schemes (#550).

data/examples/consumer-group.rb

@@ -22,6 +22,12 @@ consumer.subscribe(topic)
 trap("TERM") { consumer.stop }
 trap("INT") { consumer.stop }
 
-consumer.each_message do |message|
-  puts message.value
+begin
+  consumer.each_message do |message|
+  end
+rescue Kafka::ProcessingError => e
+  warn "Got #{e.cause}"
+  consumer.pause(e.topic, e.partition, timeout: 20)
+
+  retry
 end

data/lib/kafka/client.rb

@@ -305,12 +305,19 @@ module Kafka
         interval: heartbeat_interval,
       )
 
+      fetcher = Fetcher.new(
+        cluster: initialize_cluster,
+        logger: @logger,
+        instrumenter: instrumenter,
+      )
+
       Consumer.new(
         cluster: cluster,
         logger: @logger,
         instrumenter: instrumenter,
         group: group,
         offset_manager: offset_manager,
+        fetcher: fetcher,
         session_timeout: session_timeout,
         heartbeat: heartbeat,
       )

data/lib/kafka/consumer.rb

@@ -1,6 +1,6 @@
 require "kafka/consumer_group"
 require "kafka/offset_manager"
-require "kafka/fetch_operation"
+require "kafka/fetcher"
 
 module Kafka
 
@@ -40,13 +40,14 @@ module Kafka
   #
   class Consumer
 
-    def initialize(cluster:, logger:, instrumenter:, group:, offset_manager:, session_timeout:, heartbeat:)
+    def initialize(cluster:, logger:, instrumenter:, group:, fetcher:, offset_manager:, session_timeout:, heartbeat:)
       @cluster = cluster
       @logger = logger
       @instrumenter = instrumenter
       @group = group
       @offset_manager = offset_manager
       @session_timeout = session_timeout
+      @fetcher = fetcher
       @heartbeat = heartbeat
 
       # A list of partitions that have been paused, per topic.
@@ -55,9 +56,6 @@ module Kafka
       # Whether or not the consumer is currently consuming messages.
       @running = false
 
-      # The maximum number of bytes to fetch from a single partition, by topic.
-      @max_bytes = {}
-
       # Hash containing offsets for each topic and partition that has the
       # automatically_mark_as_processed feature disabled. Offset manager is only active
       # when everything is suppose to happen automatically. Otherwise we need to keep track of the
@@ -93,7 +91,7 @@ module Kafka
 
       @group.subscribe(topic)
       @offset_manager.set_default_offset(topic, default_offset)
-      @max_bytes[topic] = max_bytes_per_partition
+      @fetcher.subscribe(topic, max_bytes_per_partition: max_bytes_per_partition)
 
       nil
     end
@@ -136,6 +134,8 @@ module Kafka
     def resume(topic, partition)
       paused_partitions = @paused_partitions.fetch(topic, {})
       paused_partitions.delete(partition)
+
+      seek_to_next(topic, partition)
     end
 
     # Whether the topic partition is currently paused.
@@ -153,15 +153,7 @@ module Kafka
         # absolute point in time.
         timeout = partitions.fetch(partition)
 
-        if timeout.nil?
-          true
-        elsif Time.now < timeout
-          true
-        else
-          @logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
-          resume(topic, partition)
-          false
-        end
+        timeout.nil? || Time.now < timeout
       end
     end
 
@@ -193,24 +185,16 @@ module Kafka
     #   {Kafka::ProcessingError} instance.
     # @return [nil]
     def each_message(min_bytes: 1, max_bytes: 10485760, max_wait_time: 1, automatically_mark_as_processed: true)
+      @fetcher.configure(
+        min_bytes: min_bytes,
+        max_bytes: max_bytes,
+        max_wait_time: max_wait_time,
+      )
+
       consumer_loop do
-        batches = fetch_batches(
-          min_bytes: min_bytes,
-          max_bytes: max_bytes,
-          max_wait_time: max_wait_time,
-          automatically_mark_as_processed: automatically_mark_as_processed
-        )
+        batches = fetch_batches
 
         batches.each do |batch|
-          unless batch.empty?
-            @instrumenter.instrument("fetch_batch.consumer", {
-              topic: batch.topic,
-              partition: batch.partition,
-              offset_lag: batch.offset_lag,
-              highwater_mark_offset: batch.highwater_mark_offset,
-              message_count: batch.messages.count,
-            })
-          end
           batch.messages.each do |message|
             notification = {
               topic: message.topic,
@@ -281,13 +265,14 @@ module Kafka
     # @yieldparam batch [Kafka::FetchedBatch] a message batch fetched from Kafka.
     # @return [nil]
     def each_batch(min_bytes: 1, max_bytes: 10485760, max_wait_time: 1, automatically_mark_as_processed: true)
+      @fetcher.configure(
+        min_bytes: min_bytes,
+        max_bytes: max_bytes,
+        max_wait_time: max_wait_time,
+      )
+
       consumer_loop do
-        batches = fetch_batches(
-          min_bytes: min_bytes,
-          max_bytes: max_bytes,
-          max_wait_time: max_wait_time,
-          automatically_mark_as_processed: automatically_mark_as_processed
-        )
+        batches = fetch_batches
 
         batches.each do |batch|
           unless batch.empty?
@@ -369,6 +354,8 @@ module Kafka
     def consumer_loop
       @running = true
 
+      @fetcher.start
+
       while @running
         begin
           @instrumenter.instrument("loop.consumer") do
@@ -394,6 +381,8 @@ module Kafka
         end
       end
     ensure
+      @fetcher.stop
+
       # In order to quickly have the consumer group re-balance itself, it's
       # important that members explicitly tell Kafka when they're leaving.
       make_final_offsets_commit!
@@ -433,59 +422,67 @@ module Kafka
         # only keep commits for the partitions that we're still assigned.
         @offset_manager.clear_offsets_excluding(@group.assigned_partitions)
       end
-    end
-
-    def fetch_batches(min_bytes:, max_bytes:, max_wait_time:, automatically_mark_as_processed:)
-      # Return early if the consumer has been stopped.
-      return [] if !@running
-
-      join_group unless @group.member?
-
-      subscribed_partitions = @group.subscribed_partitions
 
-      @heartbeat.send_if_necessary
-
-      operation = FetchOperation.new(
-        cluster: @cluster,
-        logger: @logger,
-        min_bytes: min_bytes,
-        max_bytes: max_bytes,
-        max_wait_time: max_wait_time,
-      )
+      @fetcher.reset
 
-      subscribed_partitions.each do |topic, partitions|
+      @group.assigned_partitions.each do |topic, partitions|
         partitions.each do |partition|
-          if automatically_mark_as_processed
-            offset = @offset_manager.next_offset_for(topic, partition)
+          if paused?(topic, partition)
+            @logger.warn "Not fetching from #{topic}/#{partition} due to pause"
           else
-            # When automatic marking is off, the first poll needs to be based on the last committed
-            # offset from Kafka, that's why we fallback in case of nil (it may not be 0)
-            if @current_offsets[topic].key?(partition)
-              offset = @current_offsets[topic][partition] + 1
-            else
-              offset = @offset_manager.next_offset_for(topic, partition)
-            end
+            seek_to_next(topic, partition)
           end
+        end
+      end
+    end
 
-          max_bytes = @max_bytes.fetch(topic)
+    def seek_to_next(topic, partition)
+      # When automatic marking is off, the first poll needs to be based on the last committed
+      # offset from Kafka, that's why we fallback in case of nil (it may not be 0)
+      if @current_offsets[topic].key?(partition)
+        offset = @current_offsets[topic][partition] + 1
+      else
+        offset = @offset_manager.next_offset_for(topic, partition)
+      end
 
-          if paused?(topic, partition)
-            @logger.warn "Partition #{topic}/#{partition} is currently paused, skipping"
-          else
-            @logger.debug "Fetching batch from #{topic}/#{partition} starting at offset #{offset}"
-            operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
+      @fetcher.seek(topic, partition, offset)
+    end
+
+    def resume_paused_partitions!
+      @paused_partitions.each do |topic, partitions|
+        partitions.keys.each do |partition|
+          unless paused?(topic, partition)
+            @logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
+            resume(topic, partition)
           end
         end
       end
+    end
 
-      operation.execute
-    rescue NoPartitionsToFetchFrom
-      backoff = max_wait_time > 0 ? max_wait_time : 1
+    def fetch_batches
+      # Return early if the consumer has been stopped.
+      return [] if !@running
 
-      @logger.info "There are no partitions to fetch from, sleeping for #{backoff}s"
-      sleep backoff
+      join_group unless @group.member?
 
-      retry
+      @heartbeat.send_if_necessary
+
+      resume_paused_partitions!
+
+      if !@fetcher.data?
+        @logger.debug "No batches to process"
+        sleep 2
+        []
+      else
+        tag, message = @fetcher.poll
+
+        case tag
+        when :batches
+          message
+        when :exception
+          raise message
+        end
+      end
     rescue OffsetOutOfRange => e
       @logger.error "Invalid offset for #{e.topic}/#{e.partition}, resetting to default offset"
 

data/lib/kafka/fetcher.rb

@@ -0,0 +1,178 @@
+require "kafka/fetch_operation"
+
+module Kafka
+  class Fetcher
+    MAX_QUEUE_SIZE = 100
+
+    attr_reader :queue
+
+    def initialize(cluster:, logger:, instrumenter:)
+      @cluster = cluster
+      @logger = logger
+      @instrumenter = instrumenter
+
+      @queue = Queue.new
+      @commands = Queue.new
+      @next_offsets = Hash.new { |h, k| h[k] = {} }
+
+      # Long poll until at least this many bytes can be fetched.
+      @min_bytes = 1
+
+      # Long poll at most this number of seconds.
+      @max_wait_time = 1
+
+      # The maximum number of bytes to fetch for any given fetch request.
+      @max_bytes = 10485760
+
+      # The maximum number of bytes to fetch per partition, by topic.
+      @max_bytes_per_partition = {}
+
+      @thread = Thread.new do
+        loop while true
+      end
+
+      @thread.abort_on_exception = true
+    end
+
+    def subscribe(topic, max_bytes_per_partition:)
+      @commands << [:subscribe, [topic, max_bytes_per_partition]]
+    end
+
+    def seek(topic, partition, offset)
+      @commands << [:seek, [topic, partition, offset]]
+    end
+
+    def configure(min_bytes:, max_bytes:, max_wait_time:)
+      @commands << [:configure, [min_bytes, max_bytes, max_wait_time]]
+    end
+
+    def start
+      @commands << [:start, []]
+    end
+
+    def handle_start
+      raise "already started" if @running
+
+      @running = true
+    end
+
+    def stop
+      @commands << [:stop, []]
+    end
+
+    def reset
+      @commands << [:reset, []]
+    end
+
+    def data?
+      !@queue.empty?
+    end
+
+    def poll
+      @queue.deq
+    end
+
+    private
+
+    def loop
+      if !@commands.empty?
+        cmd, args = @commands.deq
+
+        @logger.debug "Handling fetcher command: #{cmd}"
+
+        send("handle_#{cmd}", *args)
+      elsif !@running
+        sleep 0.1
+      elsif @queue.size < MAX_QUEUE_SIZE
+        step
+      else
+        @logger.warn "Reached max fetcher queue size (#{MAX_QUEUE_SIZE}), sleeping 1s"
+        sleep 1
+      end
+    end
+
+    def handle_configure(min_bytes, max_bytes, max_wait_time)
+      @min_bytes = min_bytes
+      @max_bytes = max_bytes
+      @max_wait_time = max_wait_time
+    end
+
+    def handle_reset
+      @next_offsets.clear
+    end
+
+    def handle_stop(*)
+      @running = false
+
+      # After stopping, we need to reconfigure the topics and partitions to fetch
+      # from. Otherwise we'd keep fetching from a bunch of partitions we may no
+      # longer be assigned.
+      handle_reset
+    end
+
+    def handle_subscribe(topic, max_bytes_per_partition)
+      @logger.info "Will fetch at most #{max_bytes_per_partition} bytes at a time per partition from #{topic}"
+      @max_bytes_per_partition[topic] = max_bytes_per_partition
+    end
+
+    def handle_seek(topic, partition, offset)
+      @logger.info "Seeking #{topic}/#{partition} to offset #{offset}"
+      @next_offsets[topic][partition] = offset
+    end
+
+    def step
+      batches = fetch_batches
+
+      batches.each do |batch|
+        unless batch.empty?
+          @instrumenter.instrument("fetch_batch.consumer", {
+            topic: batch.topic,
+            partition: batch.partition,
+            offset_lag: batch.offset_lag,
+            highwater_mark_offset: batch.highwater_mark_offset,
+            message_count: batch.messages.count,
+          })
+        end
+
+        @next_offsets[batch.topic][batch.partition] = batch.last_offset + 1
+      end
+
+      @queue << [:batches, batches]
+    rescue Kafka::NoPartitionsToFetchFrom
+      @logger.warn "No partitions to fetch from, sleeping for 1s"
+      sleep 1
+    rescue Kafka::Error => e
+      @queue << [:exception, e]
+    end
+
+    def fetch_batches
+      @logger.debug "Fetching batches"
+
+      operation = FetchOperation.new(
+        cluster: @cluster,
+        logger: @logger,
+        min_bytes: @min_bytes,
+        max_bytes: @max_bytes,
+        max_wait_time: @max_wait_time,
+      )
+
+      @next_offsets.each do |topic, partitions|
+        # Fetch at most this many bytes from any single partition.
+        max_bytes = @max_bytes_per_partition[topic]
+
+        partitions.each do |partition, offset|
+          operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
+        end
+      end
+
+      operation.execute
+    rescue NoPartitionsToFetchFrom
+      backoff = @max_wait_time > 0 ? @max_wait_time : 1
+
+      @logger.info "There are no partitions to fetch from, sleeping for #{backoff}s"
+      sleep backoff
+
+      []
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.5"
+  VERSION = "0.6.0.beta1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.6.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -323,6 +323,7 @@ files:
 - lib/kafka/fetch_operation.rb
 - lib/kafka/fetched_batch.rb
 - lib/kafka/fetched_message.rb
+- lib/kafka/fetcher.rb
 - lib/kafka/gzip_codec.rb
 - lib/kafka/heartbeat.rb
 - lib/kafka/instrumenter.rb
@@ -404,9 +405,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6