kafka-rest-rb 0.1.0.alpha2 → 0.1.0.alpha3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef34042990b7684ddc6a59d1ac9ba8ee44f87d30
-  data.tar.gz: 95dd2134ff748d790ffdb5b6ab216d99d326b59c
+  metadata.gz: 62669def0b5abd426e87fd604d686d80bb7b35cb
+  data.tar.gz: 34b0f48d653206b00b66d323bf70907eb9a6235c
 SHA512:
-  metadata.gz: 246f29462bbf834ae1952c7b010bea02a7bfbd8b0613b6cbfd506755144d8dcab06291346ce6db3f5b2556ed9a0002e14257152c9403c91b8c5816c190b04d35
-  data.tar.gz: eb2a8e82e9740b6e677605b848a2e2cf70330e76670533392059689acf317dac87380014fc864f0cc666185981a56a0aa6920b69aefd965bc94c4cddff117e25
+  metadata.gz: 1d3b74192ffd3bc11cb8493c0b1688cbcf973b20f3c572ae38a9edc1024bd334d3e4e2a32a360aeb512230f186a7cc849e23dd9a608aafc2a60686ef6d3ac043
+  data.tar.gz: e382353163f1fe427a970051e50476713f1f794efdacfdf7e23ce8b201cac596489211f46020e1750a4e504cf74a50a81006dfd405ece307f4645fcc34ac8da5

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 
 *.gem
+.ruby-version

data/README.md

@@ -1,3 +1,4 @@
-# Kafka-REST client, producer/consumer DSLs and worker daemon for Ruby/Rails apps. [![CircleCI](https://circleci.com/gh/konukhov/kafka-rest-rb.svg?style=shield)](https://circleci.com/gh/konukhov/kafka-rest-rb)
+# Kafka-REST client, producer/consumer DSLs and worker daemon for Ruby/Rails apps. 
+[![Gem Version](https://badge.fury.io/rb/kafka-rest-rb.svg)](https://badge.fury.io/rb/kafka-rest-rb) [![CircleCI](https://circleci.com/gh/konukhov/kafka-rest-rb.svg?style=shield)](https://circleci.com/gh/konukhov/kafka-rest-rb)
 
-In early development stage. Come back later to see the docs!
+In early development stage. Come back later to read the docs or check specs to get the idea of what's going on!

data/kafka-rest-rb.gemspec

@@ -20,9 +20,18 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'faraday', '~> 0.9'
+  spec.add_runtime_dependency 'net-http-persistent', '~> 2.9'
   spec.add_runtime_dependency 'faraday_middleware', '~> 0.10'
+
   spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
-  spec.add_runtime_dependency 'oj', '~> 2.17'
+  spec.add_runtime_dependency 'multi_json', '~> 1.12'
+  spec.add_runtime_dependency 'connection_pool', '~> 2.2'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_runtime_dependency 'jrjackson', '~> 0.3.4'
+  else
+    spec.add_runtime_dependency 'oj', '>= 2.9'
+  end
 
   spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/kafka_rest/client/middleware.rb

@@ -1,6 +1,6 @@
 require 'faraday'
 require 'faraday_middleware/response_middleware'
-require 'oj'
+require 'multi_json'
 
 module KafkaRest
   class Client
@@ -32,7 +32,7 @@ module KafkaRest
     class JsonRequest < Faraday::Middleware
       def call(env)
         if env[:body]
-          env[:body] = Oj.dump env[:body], mode: :compat, symbol_keys: false
+          env[:body] = MultiJson.dump env[:body]
         end
 
         @app.call(env)
@@ -41,7 +41,7 @@ module KafkaRest
 
     class JsonResponse < FaradayMiddleware::ResponseMiddleware
       define_parser do |body|
-        Oj.load(body)
+        MultiJson.load(body)
       end
     end
 

data/lib/kafka_rest/client.rb

@@ -1,41 +1,54 @@
 require 'kafka_rest/client/middleware.rb'
 require 'faraday'
+require 'connection_pool'
 
 module KafkaRest
   class Client
     def initialize
-      @conn = Faraday.new(url: KafkaRest.config.url) do |c|
-        c.request :encode_json
-        c.request :default_headers, default_headers
+      @conn = ConnectionPool.new(size: 8, timeout: 5) do
+        Faraday.new(url: KafkaRest.config.url) do |c|
+          c.request :encode_json
+          c.request :default_headers, default_headers
 
-        c.response :raise_exception
-        c.response :decode_json
+          c.response :raise_exception
+          c.response :decode_json
 
-        c.adapter :net_http_persistent
+          c.adapter :net_http_persistent
+        end
       end
     end
 
+    %w(get post put delete).each do |m|
+      class_eval %Q{
+        def #{m}(*args)
+          @conn.with do |c|
+            c.#{m}(*args)
+          end
+        end
+      }
+    end
+
     # Get list of topics
     ### returns: array of topics
     def topics
-      @conn.get("/topics")
+      get("/topics")
     end
 
     # Get topic metadata by name
     ### returns: name, configs, partitions
     def topic(topic)
-      @conn.get("/topics/#{topic}")
+      get("/topics/#{topic}")
     end
 
     # Get topic's partitions
     ###
     def topic_partitions(topic)
-      @conn.get("/topics/#{topic}/partitions")
+      get("/topics/#{topic}/partitions")
     end
 
     # Get topic's partition metadata
     def topic_partition(topic, partition)
-      @conn.get("/topics/#{topic}/partitions/#{partition}")
+      get("/topics/#{topic}/partitions/#{partition}")
     end
 
     # Get messages from topic's partition.
@@ -43,7 +56,7 @@ module KafkaRest
       params[:count] ||= 1
       format = params.delete(:format) || 'binary'
 
-      @conn.get(
+      get(
         "/topics/#{topic}/partitions/#{partition}/messages",
         params,
         accept(format)
@@ -55,7 +68,7 @@ module KafkaRest
         records: records.is_a?(Array) ? records : [records]
       )
 
-      @conn.post(path, body, content_type(format))
+      post(path, body, content_type(format))
     end
     private :produce_message
 
@@ -81,21 +94,21 @@ module KafkaRest
       body['auto.commit.enable'] = params[:auto_commit_enable] == true || false
       body['format'] = params[:format] || 'json'
 
-      @conn.post("consumers/#{group_name}", body)
+      post("consumers/#{group_name}", body)
     end
 
     def consumer_commit_offsets(group_name, consumer_id)
-      @conn.post("consumers/#{group_name}/instances/#{consumer_id}/offsets")
+      post("consumers/#{group_name}/instances/#{consumer_id}/offsets")
     end
 
     def consumer_remove(group_name, consumer_id)
-      @conn.delete("consumers/#{group_name}/instances/#{consumer_id}")
+      delete("consumers/#{group_name}/instances/#{consumer_id}")
     end
 
     def consumer_consume_from_topic(group_name, consumer_id, topic, params = {})
       format = params.delete(:format) || 'json'
 
-      @conn.get(
+      get(
         "consumers/#{group_name}/instances/#{consumer_id}/topics/#{topic}",
         params,
         accept(format)
@@ -103,7 +116,7 @@ module KafkaRest
     end
 
     def brokers
-      @conn.get("/brokers")
+      get("/brokers")
     end
 
     private

data/lib/kafka_rest/version.rb

@@ -1,3 +1,3 @@
 module KafkaRest
-  VERSION = '0.1.0.alpha2'
+  VERSION = '0.1.0.alpha3'
 end

data/perf/perftest

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.load_gemspec(File.expand_path('../../kafka-rest-rb.gemspec', __FILE__))
+Bundler.require(:default)
+
+require 'kafka_rest'
+require 'securerandom'
+require 'thread'
+require 'ostruct'
+
+ZK_URL = ENV['ZOOKEEPER'] || ':2181'
+KAFKA_BOOTSTRAP_SERVER = ENV['KAFKA_BOOTSTRAP_SERVER'] || ':9092'
+
+config = {
+  send_interval: ENV['SEND_INTERVAL'] && ENV['SEND_INTERVAL'].to_i,
+  parallelism: ENV['PARALLELISM'] && ENV['PARALLELISM'].to_i,
+  msg_count: ENV['MSG_COUNT'] && ENV['MSG_COUNT'].to_i
+}
+
+KafkaRest.configure do |c|
+  c.serialization_adapter = Class.new(KafkaRest::Producer::Serialization::Adapter) do
+    def serialize(obj, opts = {})
+      obj.id
+    end
+  end
+end
+
+class Test
+  FORMATS = %w(json avro binary).map(&:to_sym)
+
+  def initialize(config)
+    @topics    = 3.times.map {|_| "topic-#{SecureRandom.uuid}" }
+    @producers = @topics.zip(FORMATS).map do |topic, format|
+      build_producer(topic, format)
+    end
+
+    @workers = []
+
+    @parallelism = config[:parallelism]     || 4
+    @msg_count   = config[:msg_count]       || 1000
+    @send_interval = config[:send_interval] || 0.1
+
+    @success_count = 0
+    @errored_count = 0
+    @produce_time = 0
+
+    @errors = {}
+
+    @mutex = Mutex.new
+  end
+
+  def run!
+    begin_time = Time.now
+
+    if RUBY_PLATFORM == 'java'
+      trap('SIGTERM') { raise SignalException, 'SIGTERM' }
+      trap('SIGHUP')  { raise SignalException, 'SIGHUP' }
+      trap('SIGINT')  { raise Interrupt }
+    end
+
+    begin
+      @topics.each { |t| create_topic!(t) }
+
+      workers = @parallelism.times.map do |_|
+        Thread.new do |t|
+          log(t, "Worker initialized")
+          processed = 0
+
+          loop do
+            begin
+              t1 = Time.now
+              @producers.sample.generate_and_send!
+              t2 = (Time.now - t1) * 1000
+
+              @mutex.synchronize do
+                @success_count += 1
+                processed = @success_count + @errored_count
+                @produce_time += t2
+              end
+            rescue => e
+              @mutex.synchronize do
+                log(t, "Error while sending", false)
+
+                puts "#{e.class}: #{e.message}"
+                e.backtrace.each do |e|
+                  puts e
+                end
+
+                @errors["#{e.class.name}: #{e.message}"] ||= 0
+                @errors["#{e.class.name}: #{e.message}"] += 1
+
+                @errored_count += 1
+                processed = @success_count + @errored_count
+              end
+            end
+            if processed >= @msg_count
+              raise StopIteration
+            else
+              sleep(@send_interval)
+            end
+          end
+        end
+      end
+
+      workers.map(&:join)
+      puts "Sent all messages"
+    ensure
+      test_time = Time.now - begin_time
+
+      @topics.each { |t| remove_topic!(t) }
+      puts "Total test time: #{test_time}s"
+      puts "Total messages: #{@msg_count}"
+      puts "Total succeeded sends: #{@success_count}"
+      puts "Total failed sends: #{@errored_count}"
+      puts "AVG produce time: #{@produce_time / (@success_count + @errored_count)}ms"
+
+      if @errors.any?
+        puts "Errors encountered:"
+        @errors.each do |e, count|
+          puts "\t#{e}: #{count} times"
+        end
+      end
+    end
+  end
+
+  private
+
+  def log(thread, msg, lock = true)
+    log = ->{ "[Worker thread-#{thread.object_id}] #{msg}" }
+
+    if lock
+      @mutex.synchronize &log
+    else
+      log.call
+    end
+  end
+
+  def run_command(cmd)
+    out = nil
+
+    IO.popen(cmd) do |io|
+      out = io.read
+    end
+
+    if $?.to_i == 0
+      yield if block_given?
+    else
+      puts "Command `#{cmd}` finished with an error."
+    end
+  end
+
+  def create_topic!(topic)
+    cmd = <<-CMD
+      kafka-topics --create \
+                 --zookeeper #{ZK_URL} \
+                 --topic #{topic} \
+                 --partitions 2 \
+                 --replication-factor 1
+    CMD
+
+    run_command(cmd) do
+      puts "Successfully created topic `#{topic}`"
+    end
+  end
+
+  def remove_topic!(topic)
+    run_command("/usr/local/bin/kafka-topics --delete --topic #{topic} --zookeeper #{ZK_URL}") do
+      puts "Successfully removed topic `#{topic}`"
+    end
+  end
+
+  def build_producer(_topic, _format)
+    Class.new do
+      include KafkaRest::Producer
+
+      topic  _topic
+      format _format
+
+      if _format == :avro
+        value_schema <<-SCHEMA
+          {
+            "name":"test_schema",
+            "type":"string"
+          }
+        SCHEMA
+      end
+
+      key :id
+
+      def self.generate_and_send!
+        self.send!(OpenStruct.new(id: SecureRandom.uuid))
+      end
+    end
+  end
+end
+
+Test.new(config).run!

metadata

@@ -1,111 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: kafka-rest-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0.alpha2
+  version: 0.1.0.alpha3
 platform: ruby
 authors:
 - Theodore Konukhov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-12 00:00:00.000000000 Z
+date: 2016-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
-  type: :runtime
+  name: faraday
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
-  name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '2.9'
+  name: net-http-persistent
+  prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  name: faraday_middleware
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.10'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :runtime
+  name: concurrent-ruby
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '2.17'
+        version: '1.12'
+  name: multi_json
+  prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  name: connection_pool
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '2.17'
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+  name: jrjackson
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.4
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.12'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
 description: Kafka-REST client, DSLs and consumer workers for Ruby.
@@ -116,9 +158,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".rspec"
-- ".travis.yml"
+- .gitignore
+- .rspec
+- .travis.yml
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -149,28 +191,29 @@ files:
 - lib/kafka_rest/version.rb
 - lib/kafka_rest/worker.rb
 - lib/kafka_rest/worker/consumer_manager.rb
+- perf/perftest
 homepage: https://github.com/konukhov/kafka-rest-rb
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
 specification_version: 4
 summary: Kafka-REST proxy client for Ruby on Rails.
 test_files: []