ruby-kafka 0.3.14 → 0.3.15.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9464a030fbaaa71d6b6095fe7519787396afbc2d
-  data.tar.gz: f432a7bf7401371251522b75980ff9116c8cb89e
+  metadata.gz: f737e31dc809145f5ea7d5d66d761df383103215
+  data.tar.gz: 82e1bb74dd93e01963c8ccff60ea4e26a27d24f3
 SHA512:
-  metadata.gz: cbedf9f17a00f597700775919f396803f80a77cca6b33ef7c0217569a2c5d05c2f4aa6e834bb005c37b1eb090b3fb4ac994e5b3aef166bd7dc6df471c866d79b
-  data.tar.gz: c9d7e0b9a37e2c081e3bbba5ba644997311b167900df4054df4e6204c456d1652fb022019662c4b3d58a86f1f56c660a0e52efefe52e5c5247ee905e3b8e97d9
+  metadata.gz: 5cca8d29dd690134e46833d3cd01e961c4c782c540f88472619edc4e9e9af6e273c5dbafbc5fd5df31856fce3f2550f2c0e3f1924372a86aa67cfbdb87120e7f
+  data.tar.gz: 269d7ff9dfbc342a8b942249889c55d8ddb9ca4a06e18003727537fd00dcda09a42e714626fa4045554de76c0a52fdafb3694b702c39446753016a1fa0cfba85

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.15.beta1
+
+- Allow pausing consumption of a partition (#268).
+
 ## v0.3.14
 
 - Automatically recover from invalid consumer checkpoints.

data/lib/kafka.rb

@@ -4,6 +4,19 @@ module Kafka
   class Error < StandardError
   end
 
+  # There was an error processing a message.
+  class ProcessingError < Error
+    attr_reader :topic, :partition, :offset
+
+    def initialize(topic, partition, offset)
+      @topic = topic
+      @partition = partition
+      @offset = offset
+
+      super()
+    end
+  end
+
   # Subclasses of this exception class map to an error code described in the
   # Kafka protocol specification.
   #

data/lib/kafka/consumer.rb

@@ -52,6 +52,9 @@ module Kafka
       @session_timeout = session_timeout
       @heartbeat = heartbeat
 
+      # A list of partitions that have been paused, per topic.
+      @paused_partitions = {}
+
       # Whether or not the consumer is currently consuming messages.
       @running = false
 
@@ -63,7 +66,7 @@ module Kafka
     #
     # Typically you either want to start reading messages from the very
     # beginning of the topic's partitions or you simply want to wait for new
-    # messages to be written. In the former case, set `start_from_beginnign`
+    # messages to be written. In the former case, set `start_from_beginning`
     # true (the default); in the latter, set it to false.
     #
     # @param topic [String] the name of the topic to subscribe to.
@@ -91,6 +94,41 @@ module Kafka
       @running = false
     end
 
+    # Pause processing of a specific topic partition.
+    #
+    # When a specific message causes the processor code to fail, it can be a good
+    # idea to simply pause the partition until the error can be resolved, allowing
+    # the rest of the partitions to continue being processed.
+    #
+    # @param topic [String]
+    # @param partition [Integer]
+    # @return [nil]
+    def pause(topic, partition)
+      @paused_partitions[topic] ||= Set.new
+      @paused_partitions[topic] << partition
+    end
+
+    # Resume processing of a topic partition.
+    #
+    # @see #pause
+    # @param topic [String]
+    # @param partition [Integer]
+    # @return [nil]
+    def resume(topic, partition)
+      paused_partitions = @paused_partitions.fetch(topic, [])
+      paused_partitions.delete(partition)
+    end
+
+    # Whether the topic partition is currently paused.
+    #
+    # @see #pause
+    # @param topic [String]
+    # @param partition [Integer]
+    # @return [Boolean] true if the partition is paused, false otherwise.
+    def paused?(topic, partition)
+      @paused_partitions.fetch(topic, []).include?(partition)
+    end
+
     # Fetches and enumerates the messages in the topics that the consumer group
     # subscribes to.
     #
@@ -132,7 +170,7 @@ module Kafka
                 backtrace = e.backtrace.join("\n")
                 @logger.error "Exception raised when processing #{location} -- #{e.class}: #{e}\n#{backtrace}"
 
-                raise
+                raise ProcessingError.new(message.topic, message.partition, message.offset)
               end
             end
 
@@ -274,9 +312,12 @@ module Kafka
           offset = @offset_manager.next_offset_for(topic, partition)
           max_bytes = @max_bytes.fetch(topic)
 
-          @logger.debug "Fetching batch from #{topic}/#{partition} starting at offset #{offset}"
-
-          operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
+          if paused?(topic, partition)
+            @logger.warn "Partition #{topic}/#{partition} is currently paused, skipping"
+          else
+            @logger.debug "Fetching batch from #{topic}/#{partition} starting at offset #{offset}"
+            operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
+          end
         end
       end
 

data/lib/kafka/offset_manager.rb

@@ -71,6 +71,7 @@ module Kafka
 
     def clear_offsets
       @processed_offsets.clear
+      @resolved_offsets.clear
 
       # Clear the cached commits from the brokers.
       @committed_offsets = nil
@@ -86,6 +87,7 @@ module Kafka
 
       # Clear the cached commits from the brokers.
       @committed_offsets = nil
+      @resolved_offsets.clear
     end
 
     private

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.14"
+  VERSION = "0.3.15.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.15.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-17 00:00:00.000000000 Z
+date: 2016-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -303,9 +303,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5.1