kafka_syrup 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 68b32dfdf15ed7d504a6c86ae20f166d9bc946a4
+  data.tar.gz: 1df748f47f3aff2b675bcd4971f1aeed6656e429
+SHA512:
+  metadata.gz: 8857d9dfdcb3df0c5364d0ce0cb1c9f1b230525576cd5959d3020d557d9396e9ce7e61dec30f55589f0535f9f6536695be61f5843857eeb2d759f6f0080bef8b
+  data.tar.gz: bcaf2bde3c43304d0531c388e86a6a92713e03a63fe0dacd8f543573ec534e2ef7baae48cc1154156e1d49bf5d7f2af36579741197c4d94e75546f6cbaac43d8

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/
+/bin/

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kafkasyrup.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Delbert Mitten
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,79 @@
+# KafkaSyrup
+
+KafkaSyrup is a Kafka client compatible with the Kafka 0.8 API and
+above.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kafka_syrup'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kafka-syrup
+
+## Usage
+
+### Configuration
+
+```ruby
+require 'kafka_syrup'
+
+KafkaSyrup.configure do |config|
+  config.brokers = 'localhost:9092,localhost:9093,localhost:9094'
+
+  # For consuming in a group
+  config.zookeeper_hosts = 'localhost:2181'
+end
+```
+
+*Additional configuration options can be found in the main
+kafka_syrup.rb file*
+
+### Sending messages to Kafka
+
+```ruby
+producer = KafkaSyrup::Producer.new(topic: :foo)
+
+producer.send_message('hello world')
+```
+
+### Consuming messages from a single Kafka partition
+
+```ruby
+consumer = KafkaSyrup::PartitionConsumer.new(topic: :foo, partition: 1)
+
+consumer.fetch
+
+# It is possible to limit the number of messages returned:
+consumer.fetch(50)
+```
+
+*Note that regardless of the limit, fetch() will block until it has received at least one
+message.*
+
+### Consuming messages in a group
+
+```ruby
+consumer = KafkaSyrup::TopicConsumer.new(topic: :foo, group: :bar)
+
+consumer.fetch
+
+# It is possible to limit the number of messages returned:
+consumer.fetch(50)
+```
+
+*Note that regardless of the limit, fetch() will block until it has received at least one
+message.*
+
+The topic consumer utilizes zookeeper for coordination with other
+members of the group and is fully compatible with the normal Kafka high
+level consumer. (ie - kafk_syrup clients and java kafka clients can
+coexist in the same group with no problem.)

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/test_*.rb'
+end

data/kafka_syrup.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kafka_syrup/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kafka_syrup"
+  spec.version       = KafkaSyrup::VERSION
+  spec.authors       = ['Delbert Mitten']
+  spec.email         = ['drmitten@gmail.com']
+
+  spec.summary       = %q{A high level Kafka client.}
+  spec.description   = %q{A high level Kafka client that supports producer, low level consumers, and high level consumers.}
+  spec.homepage      = 'https://github.com/drmitten/kafka_syrup'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'zk', '~> 1.9'
+  spec.add_dependency 'multi_json', '~> 1.8'
+
+  spec.add_development_dependency 'bundler', '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.6'
+  spec.add_development_dependency 'minitest-reporters', '~> 1.0'
+  spec.add_development_dependency 'mocha', '~> 1.1'
+
+  spec.add_development_dependency 'pry', '~> 0.10'
+end

data/lib/kafka_syrup.rb

@@ -0,0 +1,68 @@
+%w[
+  errors
+  utils
+  encoding
+  protocol
+  version
+  broker
+  topic_producer
+  partition_consumer
+  topic_consumer
+].each{ |file| require "kafka_syrup/#{file}" }
+
+module KafkaSyrup
+  E = Encoding # Just to abbreviate the typing
+
+  Configuration = Struct.new :produce_required_acks, :produce_timeout, :consume_max_wait_time, :consume_min_bytes, :consume_max_bytes, :so_sndbuf,
+   :brokers, :zookeeper_hosts, :zookeeper_path, :retry_backoff, :logger do
+
+    def with_defaults
+      self.produce_required_acks = -1
+      self.produce_timeout = 1500
+      self.consume_max_wait_time = 100
+      self.consume_min_bytes = 1
+      self.consume_max_bytes = 1024 * 1024
+      self.so_sndbuf = 100 * 1024
+      self.brokers = ''
+      self.zookeeper_path = '/'
+      self.retry_backoff = 10 * 1000
+      self
+    end
+
+    def logger
+      return @logger if @logger
+      @logger = Logger.new(STDOUT)
+      @logger.formatter = ->(severity, datetime, progname, msg){ "[#{datetime}] #{severity} : #{msg}\n" }
+      @logger.level = Logger::WARN
+      @logger
+    end
+  end
+
+  class << self
+    def configure
+      yield config
+    end
+
+    def config
+      @config ||= Configuration.new.with_defaults
+    end
+
+    def brokers
+      @brokers ||= config.brokers.split(',').map(&:strip).map{ |info| Broker.new(*info.split(':')) }
+    end
+
+    def get_metadata(*topics)
+      request = KafkaSyrup::Protocol::MetadataRequest.new(*topics)
+
+      brokers.each do |broker|
+        begin
+          response = broker.send_request(request, close: true)
+          return response
+        rescue StandardError
+        end
+      end
+
+      raise NoBrokers
+    end
+  end
+end

data/lib/kafka_syrup/broker.rb

@@ -0,0 +1,42 @@
+module KafkaSyrup
+  class Broker
+    attr_accessor :host, :port
+
+    def initialize(host, port)
+      self.host = host
+      self.port = port
+      self.extend Communications
+    end
+
+    module Communications
+      def socket
+        unless @socket.respond_to?(:closed?) && !@socket.closed?
+          @socket = Socket.new(:INET, :SOCK_STREAM)
+          @socket.setsockopt(:SOCKET, :SO_SNDBUF, KafkaSyrup.config.so_sndbuf)
+          @socket.connect(Socket.pack_sockaddr_in(port, host))
+        end
+        @socket
+      rescue => e
+        @socket.close rescue
+        @socket = nil
+        raise e
+      end
+
+      def send_request(req, opts = {}, &block)
+        begin
+          socket.write(req.encode)
+
+          response = self.class.const_get(req.class.to_s.sub(/Request$/, 'Response')).new(socket, &block)
+        rescue KafkaSyrup::KafkaResponseError => e
+          raise e
+        rescue StandardError => e
+          raise KafkaSyrup::SocketReadError.new(e)
+        end
+
+        socket.close if opts[:close]
+
+        response
+      end
+    end
+  end
+end

data/lib/kafka_syrup/encoding.rb

@@ -0,0 +1,59 @@
+module KafkaSyrup
+  module Encoding
+    class << self
+      # Integer encoding methods
+      { 8 => 'c',
+        16 => 's>',
+        32 => 'l>',
+        64 => 'q>'
+      }.each do |size, pattern|
+        define_method "write_int#{size}" do |num|
+          [num].pack(pattern)
+        end
+
+        define_method "read_int#{size}" do |io|
+          io.read(size/8).unpack(pattern).first
+        end
+      end
+
+      # String and Byte encoding methods
+      { string: 16,
+        bytes: 32
+      }.each do |type, size|
+        define_method "write_#{type}" do |val|
+          len = val.to_s.length
+          if len > 0
+            send("write_int#{size}", len) + val.to_s
+          else
+            send("write_int#{size}", -1)
+          end
+        end
+
+        define_method "read_#{type}" do |io|
+          len = send("read_int#{size}", io)
+          if len > 0
+            io.read(len)
+          end
+        end
+      end
+
+      def write_array(items, &block)
+        result  =  write_int32(items.length)
+        if block_given?
+          result += items.map(&block).join
+        else
+          items.each do |item|
+            result += item.respond_to?(:encode) ? item.encode : item
+          end
+        end
+        result
+      end
+
+      def read_array(io, &block)
+        [].tap do |result|
+          read_int32(io).times{ result << yield(io) }
+        end
+      end
+    end
+  end
+end

data/lib/kafka_syrup/errors.rb

@@ -0,0 +1,15 @@
+module KafkaSyrup
+  class Error < StandardError
+    def initialize(e = nil)
+      super
+      set_backtrace e.backtrace if e.is_a?(StandardError)
+    end
+  end
+
+  class SocketReadError < Error; end;
+  class NoBrokers < Error; end;
+  class TopicNotFound < Error; end;
+  class PartitionNotFound < Error; end;
+  class BrokerNotFound < Error; end;
+  class NotRegistered < Error; end;
+end

data/lib/kafka_syrup/partition_consumer.rb

@@ -0,0 +1,111 @@
+module KafkaSyrup
+  class PartitionConsumer
+    include Utils
+
+    attr_accessor :topic, :partition, :broker, :offset, :max_bytes, :messages, :thread, :control_queue, :lock
+
+    def initialize(*args)
+      load_args(*args)
+
+      refresh_metadata
+
+      self.messages = Queue.new
+      self.control_queue = Queue.new
+      self.lock = Mutex.new
+    end
+
+    def refresh_metadata
+      broker && broker.socket && broker.socket.close
+
+      meta = KafkaSyrup.get_metadata(topic)
+
+      self.broker = meta.brokers.detect{ |b| b.node == partition_from_response(meta).leader }
+      raise BrokerNotFound unless self.broker
+      self.broker.extend KafkaSyrup::Broker::Communications
+    end
+
+    def get_available_offset(time = :latest)
+      request = KafkaSyrup::Protocol::OffsetRequest.new
+      request.add_topic(topic).add_partition(partition, time == :earliest ? -2 : -1)
+
+      response = broker.send_request(request)
+      partition_from_response(response).offsets.last
+    end
+
+    def fetch_from_broker(&block)
+      lock.synchronize{ self.offset = get_available_offset(offset) } unless offset.is_a?(Fixnum)
+
+      opts = { max_bytes: max_bytes } if max_bytes
+      request = KafkaSyrup::Protocol::FetchRequest.new(opts)
+      request.add_topic(topic).add_partition(partition, offset)
+
+      response = partition_from_response(broker.send_request(request, &block))
+
+      lock.synchronize{ self.offset = response.messages.last.offset + 1 } unless response.messages.empty?
+    rescue KafkaSyrup::KafkaResponseErrors::OffsetOutOfRange
+      low = get_available_offset(:earliest)
+      high = get_available_offset
+
+      lock.synchronize{ self.offset = offset < low ? low : high }
+    end
+
+    def fetch(limit = nil)
+      start_fetcher_thread unless thread
+
+      control_queue.push(:fetch) if messages.empty? && control_queue.num_waiting > 0
+
+      result = []
+
+      loop do
+        result << messages.pop
+        break if messages.empty? || (limit && result.count == limit)
+      end
+
+      self.offset = result.last[:offset] + 1
+
+      result
+    end
+
+    def retry_backoff
+      @retry_backoff ||= KafkaSyrup.config.retry_backoff / 1000.0
+    end
+
+    private
+
+    def partition_from_response(response)
+      topic_meta = response.topics.detect{ |t| t.name == topic.to_s }
+      raise TopicNotFound unless topic_meta
+      partition_meta = topic_meta.partitions.detect{ |p| p.id == partition }
+      raise PartitionNotFound unless partition_meta
+      partition_meta
+    end
+
+    def start_fetcher_thread
+      self.thread = Thread.new do
+        log.debug "Starting Fetcher Thread for partition #{partition}"
+        loop do
+          begin
+            control_queue.pop # wait for start message
+            log.debug "Fetching from partition #{partition}"
+
+            num_received = 0
+            begin
+              fetch_from_broker do |msg|
+                messages.push partition: partition, offset: msg.offset, message: msg.value
+                num_received += 1
+              end
+
+              # No messages received so backoff a bit before retrying
+              sleep retry_backoff if num_received == 0
+            rescue
+              sleep retry_backoff
+            end
+          rescue
+            sleep retry_backoff
+          end
+        end
+      end
+      sleep 0.1 # Slight sleep to let the thread start waiting on the control queue (avoids deadlock)
+    end
+  end
+end