rafka 0.0.10 → 0.1.0

data/README.md

@@ -1,13 +1,30 @@
 rafka-rb: Ruby driver for Rafka
 ===============================================================================
+[![Build Status](https://api.travis-ci.org/skroutz/rafka-rb.svg?branch=master)](https://travis-ci.org/skroutz/rafka-rb)
 [![Gem Version](https://badge.fury.io/rb/rafka.svg)](https://badge.fury.io/rb/rafka-rb)
 [![Documentation](http://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/github/skroutz/rafka-rb)
 
-rafka-rb is a thin Ruby client library for [Rafka](https://github.com/skroutz/rafka),
-providing a consumer and a producer with simple semantics. It is backed by
-[redis-rb](https://github.com/redis/redis-rb).
+rafka-rb is a Ruby client for [Rafka](https://github.com/skroutz/rafka),
+providing consumer and producer implementations with simple semantics.
+It is backed by [redis-rb](https://github.com/redis/redis-rb).
+
+Refer to the [API documentation](http://www.rubydoc.info/github/skroutz/rafka-rb)
+for more information.
+
+
+
+
+
+
+Features
+-------------------------------------------------------------------------------
+
+- Consumer implementation
+  - consumer groups
+  - offsets may be managed automatically or manually
+- Producer implementation
+  - support for partition hashing key
 
-View the [API documentation](http://www.rubydoc.info/github/skroutz/rafka-rb).
 
 
 
@@ -40,17 +57,19 @@ Usage
 ### Producer
 
 ```ruby
-require "rafka"
+producer = Rafka::Producer.new(host: "localhost", port: 6380)
+producer.produce("greetings", "Hello there!")
+```
+
+Refer to the [Producer API documentation](http://www.rubydoc.info/github/skroutz/rafka-rb/Rafka/Producer)
+for more information.
+
+
+
+
 
-prod = Rafka::Producer.new(host: "localhost", port: 6380)
 
-# Produce to topic "greetings". The message will be assigned to a random partition.
-prod.produce("greetings", "Hello there!")
 
-# Produce using a key. Two or more messages with the same key will always be assigned to the same partition.
-prod.produce("greetings", "Hello there!", key: "hi")
-prod.produce("greetings", "Hi there!", key: "hi")
-```
 
 
 
@@ -58,14 +77,66 @@ prod.produce("greetings", "Hi there!", key: "hi")
 ### Consumer
 
 ```ruby
-require "rafka"
-
-cons = Rafka::Consumer.new(topic: "greetings", group: "myapp", id: "greeter1")
-cons.consume # => "Hello there!"
+consumer = Rafka::Consumer.new(topic: "greetings", group: "myapp")
+msg = consumer.consume
+msg.value # => "Hello there!"
 
 # with a block
-cons.consume { |msg| puts "Received: #{msg.value}" } # => "Hello there!"
+consumer.consume { |msg| puts "Received: #{msg.value}" } # => "Hello there!"
+```
+
+Offsets are managed automatically by default. If you need more control you can
+turn off the feature and manually commit offsets:
+
+```ruby
+consumer = Rafka::Consumer.new(topic: "greetings", group: "myapp", auto_offset_commit: false)
+
+# commit a single offset
+msg = consumer.consume
+consumer.commit(msg) # => true
+
+# or commit a bunch of offsets
+msg1 = consumer.consume
+msg2 = consumer.consume
+consumer.commit(msg1, msg2) # => true
+```
+
+Refer to the [Consumer API documentation](http://www.rubydoc.info/github/skroutz/rafka-rb/Rafka/Consumer)
+for more information.
+
+
+
+
+
+
+
+
+
+
+
+Development
+-------------------------------------------------------------------------------
+
+Running Rubocop:
+
+```shell
+$ bundle exec rake rubocop
 ```
 
-`Rafka::Consumer#consume` automatically commits the offsets when the given block
-is executed without raising any exceptions.
+Unit tests run as follows:
+
+```shell
+$ bundle exec rake test
+```
+
+
+rafka-rb is indirectly tested by [Rafka's end-to-end tests](https://github.com/skroutz/rafka/tree/master/test).
+
+
+
+
+
+
+License
+-------------------------------------------------------------------------------
+rafka-rb is released under the GNU General Public License version 3. See [COPYING](COPYING).

data/Rakefile

@@ -0,0 +1,22 @@
+begin
+  require "bundler/setup"
+rescue LoadError
+  puts "You must `gem install bundler` and `bundle install` to run rake tasks"
+end
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = true
+end
+
+require "yard"
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ["lib/**/*.rb"]
+end
+
+require "rubocop/rake_task"
+RuboCop::RakeTask.new
+
+task default: [:test, :rubocop]

data/lib/rafka.rb

@@ -9,23 +9,18 @@ require "rafka/consumer"
 require "rafka/producer"
 
 module Rafka
-  DEFAULTS = {
+  REDIS_DEFAULTS = {
     host: "localhost",
     port: 6380,
-    reconnect_attempts: 5,
-  }
+    reconnect_attempts: 5
+  }.freeze
 
   def self.wrap_errors
     yield
   rescue Redis::CommandError => e
-    case
-    when e.message.start_with?("PROD ")
-      raise ProduceError, e.message[5..-1]
-    when e.message.start_with?("CONS ")
-      raise ConsumeError, e.message[5..-1]
-    else
-      raise CommandError, e.message
-    end
+    raise ProduceError, e.message[5..-1] if e.message.start_with?("PROD ")
+    raise ConsumeError, e.message[5..-1] if e.message.start_with?("CONS ")
+    raise CommandError, e.message
   end
 
   # redis-rb until 3.2.1 didn't retry to connect on
@@ -34,7 +29,7 @@ module Rafka
   #
   # TODO(agis): get rid of this method when we go to 3.2.1 or later, because
   # https://github.com/redis/redis-rb/pull/476/
-  def self.with_retry(times: DEFAULTS[:reconnect_attempts], every_sec: 1)
+  def self.with_retry(times: REDIS_DEFAULTS[:reconnect_attempts], every_sec: 1)
     attempts = 0
 
     begin
@@ -52,4 +47,3 @@ module Rafka
     end
   end
 end
-

data/lib/rafka/consumer.rb

@@ -1,15 +1,20 @@
-require 'securerandom'
+require "securerandom"
 
 module Rafka
+  # A Kafka consumer that consumes messages from a given Kafka topic
+  # and belongs to a specific consumer group. Offsets may be commited
+  # automatically or manually (see {#consume}).
+  #
+  # @see https://kafka.apache.org/documentation/#consumerapi
   class Consumer
     include GenericCommands
 
-    REQUIRED = [:group, :topic]
+    REQUIRED_OPTS = [:group, :topic].freeze
 
-    # The underlying Redis client object
+    # @return [Redis::Client] the underlying Redis client instance
     attr_reader :redis
 
-    # Create a new client instance.
+    # Initialize a new consumer.
     #
     # @param [Hash] opts
     # @option opts [String] :host ("localhost") server hostname
@@ -17,29 +22,50 @@ module Rafka
     # @option opts [String] :topic Kafka topic to consume (required)
     # @option opts [String] :group Kafka consumer group name (required)
     # @option opts [String] :id (random) Kafka consumer id
-    # @option opts [Hash] :redis ({}) Optional configuration for the
-    #   underlying Redis client
+    # @option opts [Boolean] :auto_commit (true) automatically commit
+    #   offsets
+    # @option opts [Hash] :redis ({}) Configuration for the
+    #   underlying Redis client (see {REDIS_DEFAULTS})
+    #
+    # @raise [RuntimeError] if a required option was not provided
+    #   (see {REQUIRED_OPTS})
+    #
+    # @return [Consumer]
     def initialize(opts={})
-      @options = parse_opts(opts)
-      @redis = Redis.new(@options)
-      @topic = "topics:#{opts[:topic]}"
+      opts[:id] ||= SecureRandom.hex
+      opts[:id] = "#{opts[:group]}:#{opts[:id]}"
+      opts[:auto_commit] = true if opts[:auto_commit].nil?
+
+      @rafka_opts, @redis_opts = parse_opts(opts)
+      @redis = Redis.new(@redis_opts)
+      @topic = "topics:#{@rafka_opts[:topic]}"
     end
 
-    # Fetches the next message. Offsets are commited automatically. In the
-    # block form, the offset is commited only if the given block haven't
-    # raised any exceptions.
+    # Consumes the next message.
+    #
+    # If :auto_commit is true, offsets are commited automatically.
+    # In the block form, offsets are commited only if the block executes
+    # without raising any exceptions.
     #
-    # @param timeout [Fixnum] the time in seconds to wait for a message
+    # If :auto_commit is false, offsets have to be commited manually using
+    # {#commit}.
+    #
+    # @param timeout [Fixnum] the time in seconds to wait for a message. If
+    #   reached, {#consume} returns nil.
+    #
+    # @yieldparam [Message] msg the consumed message
     #
     # @raise [MalformedMessageError] if the message cannot be parsed
+    # @raise [ConsumeError] if there was any error consuming a message
     #
-    # @return [nil, Message]
+    # @return [nil, Message] the consumed message, or nil of there wasn't any
     #
     # @example Consume a message
-    #   puts consume(5).value
+    #   msg = consumer.consume
+    #   msg.value # => "hi"
     #
-    # @example Consume and commit offset if the block runs successfully
-    #   consume(5) { |msg| puts "I received #{msg.value}" }
+    # @example Consume a message and commit offset if the block does not raise an exception
+    #   consumer.consume { |msg| puts "I received #{msg.value}" }
     def consume(timeout=5)
       # redis-rb didn't automatically call `CLIENT SETNAME` until v3.2.2
       # (https://github.com/redis/redis-rb/issues/510)
@@ -57,7 +83,7 @@ module Rafka
 
       begin
         Rafka.wrap_errors do
-          Rafka.with_retry(times: @options[:reconnect_attempts]) do
+          Rafka.with_retry(times: @redis_opts[:reconnect_attempts]) do
             msg = @redis.blpop(@topic, timeout: timeout)
           end
         end
@@ -90,28 +116,78 @@ module Rafka
 
       msg
     ensure
-      if msg && !raised
-        Rafka.wrap_errors do
-          @redis.rpush("acks", "#{msg.topic}:#{msg.partition}:#{msg.offset}")
+      if msg && !raised && @rafka_opts[:auto_commit]
+        commit(msg)
+      end
+    end
+
+    # Commit offsets for the given messages.
+    #
+    # If more than one messages refer to the same topic/partition pair,
+    # only the largest offset amongst them is committed.
+    #
+    # @note This is non-blocking operation; a successful server reply means
+    #   offsets are received by the server and will _eventually_ be committed
+    #   to Kafka.
+    #
+    # @param msgs [Array<Message>] the messages for which to commit offsets
+    #
+    # @raise [ConsumeError] if there was any error commiting offsets
+    #
+    # @return [Hash] the actual offsets sent for commit
+    # @return [Hash{String=>Hash{Integer=>Integer}}] the actual offsets sent
+    #   for commit.Keys denote the topics while values contain the
+    #   partition=>offset pairs.
+    def commit(*msgs)
+      tp = prepare_for_commit(*msgs)
+
+      tp.each do |topic, po|
+        po.each do |partition, offset|
+          Rafka.wrap_errors do
+            @redis.rpush("acks", "#{topic}:#{partition}:#{offset}")
+          end
         end
       end
+
+      tp
     end
 
     private
 
-    # @return [Hash]
+    # @param opts [Hash] options hash as passed to {#initialize}
+    #
+    # @return [Array<Hash, Hash>] rafka opts, redis opts
     def parse_opts(opts)
-      REQUIRED.each do |opt|
+      REQUIRED_OPTS.each do |opt|
         raise "#{opt.inspect} option not provided" if opts[opt].nil?
       end
 
       rafka_opts = opts.reject { |k| k == :redis }
-      redis_opts = opts[:redis] || {}
 
-      options = DEFAULTS.dup.merge(rafka_opts).merge(redis_opts)
-      options[:id] = SecureRandom.hex if options[:id].nil?
-      options[:id] = "#{options[:group]}:#{options[:id]}"
-      options
+      redis_opts = REDIS_DEFAULTS.dup.merge(opts[:redis] || {})
+      redis_opts.merge!(
+        rafka_opts.select { |k| [:host, :port, :id].include?(k) }
+      )
+
+      [rafka_opts, redis_opts]
+    end
+
+    # Accepts one or more messages and prepare them for commit.
+    #
+    # @param msgs [Array<Message>]
+    #
+    # @return [Hash{String=>Hash{Integer=>Integer}}] the offsets to be commited.
+    #   Keys denote the topics while values contain the partition=>offset pairs.
+    def prepare_for_commit(*msgs)
+      tp = Hash.new { |h, k| h[k] = Hash.new(0) }
+
+      msgs.each do |msg|
+        if msg.offset >= tp[msg.topic][msg.partition]
+          tp[msg.topic][msg.partition] = msg.offset
+        end
+      end
+
+      tp
     end
   end
 end

data/lib/rafka/message.rb

@@ -1,11 +1,19 @@
 module Rafka
   # Message represents a message consumed from a topic.
   class Message
-    attr :topic, :partition, :offset, :value
+    attr_reader :topic, :partition, :offset, :value
 
+    # @param msg [Array] a message as received by the server
+    #
+    # @raise [MalformedMessageError] if message is malformed
+    #
+    # @example
+    #   Message.new(
+    #     ["topic", "greetings", "partition", 2, "offset", 321123, "value", "Hi!"]
+    #   )
     def initialize(msg)
       if !msg.is_a?(Array) || msg.size != 8
-        raise MalformedMessageError.new(msg)
+        raise MalformedMessageError, msg
       end
 
       @topic = msg[1]
@@ -14,7 +22,7 @@ module Rafka
         @partition = Integer(msg[3])
         @offset = Integer(msg[5])
       rescue ArgumentError
-        raise MalformedMessageError.new(msg)
+        raise MalformedMessageError, msg
       end
 
       @value = msg[7]

data/lib/rafka/producer.rb

@@ -1,16 +1,20 @@
 module Rafka
+  # A Kafka producer that can produce to different topics.
+  # See {#produce} for more info.
+  #
+  # @see https://kafka.apache.org/documentation/#producerapi
   class Producer
     include GenericCommands
 
     # Access the underlying Redis client object
     attr_reader :redis
 
-    # Create a new client instance.
+    # Create a new producer.
     #
     # @param [Hash] opts
     # @option opts [String] :host ("localhost") server hostname
     # @option opts [Fixnum] :port (6380) server port
-    # @options opts [Hash] :redis Configuration options for the underlying
+    # @option opts [Hash] :redis Configuration options for the underlying
     #   Redis client
     #
     # @return [Producer]
@@ -19,18 +23,21 @@ module Rafka
       @redis = Redis.new(@options)
     end
 
-    # Produce a message. This is an asynchronous operation.
+    # Produce a message to a topic. This is an asynchronous operation.
     #
     # @param topic [String]
-    # @param msg [#to_s]
-    # @param key [#to_s] two or more messages with the same key will always be
-    #   assigned to the same partition.
+    # @param msg [#to_s] the message
+    # @param key [#to_s] an optional partition hashing key. Two or more messages
+    #   with the same key will always be written to the same partition.
     #
-    # @example
-    #   produce("greetings", "Hello there!")
+    # @example Simple produce
+    #   producer = Rafka::Producer.new
+    #   producer.produce("greetings", "Hello there!")
     #
-    # @example
-    #   produce("greetings", "Hello there!", key: "hi")
+    # @example Produce two messages with a hashing key. Those messages are guaranteed to be written to the same partition
+    #     producer = Rafka::Producer.new
+    #     produce("greetings", "Aloha", key: "abc")
+    #     produce("greetings", "Hola", key: "abc")
     def produce(topic, msg, key: nil)
       Rafka.wrap_errors do
         Rafka.with_retry(times: @options[:reconnect_attempts]) do
@@ -41,10 +48,10 @@ module Rafka
       end
     end
 
-    # Flush any buffered messages. Blocks until all messages are flushed or
-    # timeout exceeds.
+    # Flush any buffered messages. Blocks until all messages are written or the
+    # given timeout exceeds.
     #
-    # @param timeout_ms [Fixnum] (5000) The timeout in milliseconds
+    # @param timeout_ms [Fixnum]
     #
     # @return [Fixnum] The number of unflushed messages
     def flush(timeout_ms=5000)
@@ -59,7 +66,7 @@ module Rafka
     def parse_opts(opts)
       rafka_opts = opts.reject { |k| k == :redis }
       redis_opts = opts[:redis] || {}
-      DEFAULTS.dup.merge(opts).merge(redis_opts)
+      REDIS_DEFAULTS.dup.merge(opts).merge(redis_opts).merge(rafka_opts)
     end
   end
 end