kafka-rest-rb 0.1.0.alpha3 → 0.1.0.alpha4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62669def0b5abd426e87fd604d686d80bb7b35cb
-  data.tar.gz: 34b0f48d653206b00b66d323bf70907eb9a6235c
+  metadata.gz: 0e943787c23ecd8a91ccbe8dd0af4da7b5cc4d6d
+  data.tar.gz: c067af79eca968aa0400006ec11af42afc5cb868
 SHA512:
-  metadata.gz: 1d3b74192ffd3bc11cb8493c0b1688cbcf973b20f3c572ae38a9edc1024bd334d3e4e2a32a360aeb512230f186a7cc849e23dd9a608aafc2a60686ef6d3ac043
-  data.tar.gz: e382353163f1fe427a970051e50476713f1f794efdacfdf7e23ce8b201cac596489211f46020e1750a4e504cf74a50a81006dfd405ece307f4645fcc34ac8da5
+  metadata.gz: 29a226688e4184daa8fbb636c7e6e2f59631faddfacb15d3c162cd315f1c6b38bf6fe8d683ea35fc42f0af457e79dadec48341223caa4e1eb2f5cb95a4fc7eef
+  data.tar.gz: 0ec544e2efd18df718d9676baaeafea0773ac8c86b0f526d89e87b4f8f568c4efeaf49875f0da333148a0dcaa3fc4ec68d430623416c384ee9ed1840e31b6b07

data/lib/kafka_rest/client/middleware.rb

@@ -4,19 +4,6 @@ require 'multi_json'
 
 module KafkaRest
   class Client
-    class KafkaRestClientException < StandardError
-      attr_reader :body, :status
-
-      def initialize(resp)
-        @body = resp.body
-        @status = resp.status
-
-        super "#{@body['message']}" +
-              " (HTTP Status: #{@status}; " +
-              "error code: #{@body['error_code']})"
-      end
-    end
-
     class DefaultHeaders < Faraday::Middleware
       def initialize(app = nil, default_headers = {})
         @default_headers = default_headers
@@ -50,7 +37,7 @@ module KafkaRest
         response = @app.call(env)
         response.on_complete do
           unless response.success?
-            raise KafkaRestClientException.new(response)
+            raise ClientError.new(response)
           end
         end
       end

data/lib/kafka_rest/config.rb

@@ -5,7 +5,8 @@ module KafkaRest
                   :serialization_adapter,
                   :worker_min_threads,
                   :worker_max_threads,
-                  :worker_max_queue
+                  :worker_max_queue,
+                  :sender
 
     def initialize
       @url = 'http://localhost:8082'
@@ -14,6 +15,15 @@ module KafkaRest
       @worker_min_threads = 4
       @worker_max_threads = 4
       @worker_max_queue = nil
+      @sender = KafkaRest::Sender::KafkaSender
+    end
+
+    def sender=(_sender)
+      if s.respond_to?(:send!)
+        @sender = _sender
+      else
+        raise InvalidConfigValue.new("sender", _sender, "Sender be a child of `KafkaRest::Sender`")
+      end
     end
   end
 

data/lib/kafka_rest/exceptions.rb

@@ -0,0 +1,29 @@
+module KafkaRest
+  class InvalidConfigValue < StandardError
+    def initialize(name, val, msg = nil)
+      message = "Invalid config for `#{name}`: #{val.to_s}"
+      message << ". #{msg}" if msg
+
+      super message
+    end
+  end
+
+  class ClientError < StandardError
+    attr_reader :body, :status, :error_code
+
+    def initialize(resp)
+      @body = resp.body
+      @status = resp.status
+      @error_code = @body['error_code']
+
+      super "#{@body['message']}" +
+            " (HTTP Status: #{@status}; " +
+            "error code: #{@status})"
+    end
+  end
+
+  class ProducerSendError < StandardError
+  end
+
+  class ConsumerError < StandardError; end
+end

data/lib/kafka_rest/producer/message.rb

@@ -0,0 +1,27 @@
+module KafkaRest
+  module Producer
+    class Message
+      attr_reader :topic, :payload, :format, :params
+
+      def initialize(producer, obj, opts = {})
+        @topic    = producer.get_topic.to_s
+        @payload  = Payload.new(producer, obj, opts).build
+        @format   = producer.get_format.to_s
+        @params   = build_params(producer)
+      end
+
+      private
+
+      # add schemas if format == 'avro'
+      def build_params(producer)
+        {}.tap do |params|
+          if @format == 'avro'
+            has_key = !producer.get_key.nil?
+            params[:key_schema] = producer.get_key_schema if has_key
+            params[:value_schema] = producer.get_value_schema
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka_rest/{sender → producer}/payload/avro_builder.rb

@@ -1,5 +1,5 @@
 module KafkaRest
-  class Sender
+  module Producer
     class Payload
       class AvroBuilder < Builder
         # TODO: get rid of this

data/lib/kafka_rest/{sender → producer}/payload/binary_builder.rb

@@ -1,7 +1,7 @@
 require 'base64'
 
 module KafkaRest
-  class Sender
+  module Producer
     class Payload
       class BinaryBuilder < Builder
         def build

data/lib/kafka_rest/{sender → producer}/payload/builder.rb

@@ -1,5 +1,5 @@
 module KafkaRest
-  class Sender
+  module Producer
     class Payload
       class Builder
         def initialize(payload)

data/lib/kafka_rest/{sender → producer}/payload/json_builder.rb

@@ -1,5 +1,5 @@
 module KafkaRest
-  class Sender
+  module Producer
     class Payload
       class JsonBuilder < Builder
         def build

data/lib/kafka_rest/{sender → producer}/payload.rb

@@ -1,10 +1,10 @@
-require 'kafka_rest/sender/payload/builder'
-require 'kafka_rest/sender/payload/avro_builder'
-require 'kafka_rest/sender/payload/json_builder'
-require 'kafka_rest/sender/payload/binary_builder'
+require 'kafka_rest/producer/payload/builder'
+require 'kafka_rest/producer/payload/avro_builder'
+require 'kafka_rest/producer/payload/json_builder'
+require 'kafka_rest/producer/payload/binary_builder'
 
 module KafkaRest
-  class Sender
+  module Producer
     class Payload
       attr_reader :klass
 

data/lib/kafka_rest/producer.rb

@@ -1,4 +1,5 @@
 require 'kafka_rest/dsl'
+require 'kafka_rest/producer/payload'
 
 module KafkaRest
   module Producer
@@ -60,9 +61,15 @@ module KafkaRest
     end
 
     module ClassMethods
-      def send!(obj, opts = {}, producer = nil)
-        (producer || KafkaRest::Sender.instance)
-          .send!(self, obj, opts)
+      def build_message(obj, opts = {})
+        Message.new(self, obj, opts = {})
+      end
+
+      def send!(obj, opts = {}, sender = nil)
+        sender  = sender || KafkaRest.config.sender
+        message = build_message(obj, opts)
+
+        sender.send!(message)
       end
     end
   end

data/lib/kafka_rest/sender/kafka_sender.rb

@@ -0,0 +1,84 @@
+module KafkaRest
+  module Sender
+    class KafkaSender
+
+      @@instance = nil
+
+      class << self
+        def instance
+          @@instance ||= self.new(Client.new)
+        end
+
+        def send!(message)
+          instance.send!(message)
+        end
+      end
+
+      attr_reader :key_schema_cache, :value_schema_cache
+
+      def initialize(client, opts = {})
+        @lock = Mutex.new
+        @client = client
+        @key_schema_cache = {}
+        @value_schema_cache = {}
+      end
+
+      # TODO: back-off retry if offset[i].errors is a retriable error
+      def send!(message)
+        send_produce_request!(
+          message.topic,
+          message.payload,
+          message.format,
+          build_params(message)
+        )
+      end
+
+      private
+
+      # replace key and value schemas with
+      # their ids if those are found in cache
+      def build_params(message)
+        return message.params unless message.format == 'avro'
+
+        topic, params, format = message.topic, message.params, message.format
+
+        {}.tap do |_p|
+          if format == 'avro'
+            if key_schema = params[:key_schema]
+              if key_schema_id = @key_schema_cache[topic]
+                _p[:key_schema_id] = key_schema_id
+              else
+                _p[:key_schema] = key_schema
+              end
+            end
+
+            if value_schema_id = @value_schema_cache[topic]
+              _p[:value_schema_id] = value_schema_id
+            else
+              _p[:value_schema] = params[:value_schema]
+            end
+          end
+        end
+      end
+
+      def send_produce_request!(topic, payload, format, params)
+        @client.topic_produce_message(topic, payload, format, params).body.tap do |re|
+          # this too (line 27)
+          cache_schema_ids!(re, topic) if format == 'avro'
+        end['offsets']
+      end
+
+      def cache_schema_ids!(resp, topic)
+        @lock.synchronize do
+          if @key_schema_cache[topic].nil? && kid = resp['key_schema_id']
+            @key_schema_cache[topic] = kid
+          end
+
+          if @value_schema_cache[topic].nil? && vid = resp['value_schema_id']
+            @value_schema_cache[topic] = vid
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka_rest/sender/log_sender.rb

@@ -0,0 +1,13 @@
+module KafkaRest
+  module Sender
+    class LogSender
+      def initialize(logger = nil)
+        @logger = logger || Logging.logger
+      end
+
+      def send!(msg)
+        @logger.info msg # todo
+      end
+    end
+  end
+end

data/lib/kafka_rest/sender/test_sender.rb

@@ -0,0 +1,28 @@
+module KafkaRest
+  module Sender
+    class TestSender
+      attr_reader :messages
+
+      def initialize
+        @lock     = Mutex.new
+        @messages = {}
+      end
+
+      def send!(message)
+        @lock.synchronize do
+          topic = message.topic
+          @messages[topic] ||= []
+          @messages[topic] << message
+        end
+      end
+
+      def last_for(topic)
+        @messages[topic].last
+      end
+
+      def reset!
+        @messages = {}
+      end
+    end
+  end
+end

data/lib/kafka_rest/sender.rb

@@ -1,81 +1,13 @@
 require 'thread'
-require 'kafka_rest/sender/payload'
 
 module KafkaRest
-  class Sender
-    @@lock = Mutex.new
+  module Sender
+    require 'kafka_rest/sender/kafka_sender'
+    require 'kafka_rest/sender/log_sender'
+    require 'kafka_rest/sender/test_sender'
 
-    class << self
-      def instance
-        @@lock.synchronize do
-          @instance ||= self.new(Client.new, lock: @@lock)
-        end
-      end
-    end
-
-    attr_reader :key_schema_cache, :value_schema_cache
-
-    # TODO: buffering???
-    def initialize(client, opts = {})
-      @lock = opts[:lock] || Mutex.new
-      @client = client
-      @key_schema_cache = {}
-      @value_schema_cache = {}
-    end
-
-    # TODO: back-off retry if offset[i].errors is a retriable error
-    def send!(klass, obj, opts = {})
-      topic, payload, format, params = build_request(klass, obj, opts)
-      send_produce_request!(topic, payload, format, params)
-    end
-
-    private
-
-    def build_request(klass, obj, opts)
-      # TODO: oooh, dirty and weird - this should not be here.
-      #       come up with something good!
-      topic    = klass.get_topic.to_s
-      key      = klass.get_key
-      payload  = Payload.new(klass, obj, opts).build
-      format   = klass.get_format.to_s
-      params   = {}.tap do |_p|
-        if format == 'avro'
-          unless key.nil?
-            if kid = @key_schema_cache[topic]
-              _p[:key_schema_id] = kid
-            else
-              _p[:key_schema] = klass.get_key_schema
-            end
-          end
-
-          if vid = @value_schema_cache[topic]
-            _p[:value_schema_id] = vid
-          else
-            _p[:value_schema] = klass.get_value_schema
-          end
-        end
-      end
-
-      [topic, payload, format, params]
-    end
-
-    def send_produce_request!(topic, payload, format, params)
-      @client.topic_produce_message(topic, payload, format, params).body.tap do |re|
-        # this too (line 27)
-        cache_schema_ids!(re, topic) if format == 'avro'
-      end['offsets']
-    end
-
-    def cache_schema_ids!(resp, topic)
-      @lock.synchronize do
-        if @key_schema_cache[topic].nil? && kid = resp['key_schema_id']
-          @key_schema_cache[topic] = kid
-        end
-
-        if @value_schema_cache[topic].nil? && vid = resp['value_schema_id']
-          @value_schema_cache[topic] = vid
-        end
-      end
+    def self.valid?(sender)
+      sender.respond_to? :send!
     end
   end
 end

data/lib/kafka_rest/version.rb

@@ -1,3 +1,3 @@
 module KafkaRest
-  VERSION = '0.1.0.alpha3'
+  VERSION = '0.1.0.alpha4'
 end

data/lib/kafka_rest.rb

@@ -1,10 +1,14 @@
+module KafkaRest; end
+
+require 'kafka_rest/exceptions'
+require 'kafka_rest/sender'
 require 'kafka_rest/config'
 require 'kafka_rest/logging'
 require 'kafka_rest/client'
 require 'kafka_rest/worker'
 require 'kafka_rest/producer'
+require 'kafka_rest/producer/message'
 require 'kafka_rest/producer/serialization/adapter'
-require 'kafka_rest/sender'
 require 'kafka_rest/consumer'
 
 KafkaRest.configure do |c|

metadata

@@ -1,153 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: kafka-rest-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0.alpha3
+  version: 0.1.0.alpha4
 platform: ruby
 authors:
 - Theodore Konukhov
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-18 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-  name: faraday
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
+  name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
-  name: net-http-persistent
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
 - !ruby/object:Gem::Dependency
+  name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
-  name: faraday_middleware
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
-  name: multi_json
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
+  name: connection_pool
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
-  name: connection_pool
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
 - !ruby/object:Gem::Dependency
+  name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
-  name: jrjackson
-  prerelease: false
+        version: '2.9'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: '2.9'
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 description: Kafka-REST client, DSLs and consumer workers for Ruby.
@@ -158,9 +158,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -177,17 +177,22 @@ files:
 - lib/kafka_rest/config.rb
 - lib/kafka_rest/consumer.rb
 - lib/kafka_rest/dsl.rb
+- lib/kafka_rest/exceptions.rb
 - lib/kafka_rest/logging.rb
 - lib/kafka_rest/producer.rb
+- lib/kafka_rest/producer/message.rb
+- lib/kafka_rest/producer/payload.rb
+- lib/kafka_rest/producer/payload/avro_builder.rb
+- lib/kafka_rest/producer/payload/binary_builder.rb
+- lib/kafka_rest/producer/payload/builder.rb
+- lib/kafka_rest/producer/payload/json_builder.rb
 - lib/kafka_rest/producer/serialization/active_model.rb
 - lib/kafka_rest/producer/serialization/adapter.rb
 - lib/kafka_rest/producer/serialization/noop.rb
 - lib/kafka_rest/sender.rb
-- lib/kafka_rest/sender/payload.rb
-- lib/kafka_rest/sender/payload/avro_builder.rb
-- lib/kafka_rest/sender/payload/binary_builder.rb
-- lib/kafka_rest/sender/payload/builder.rb
-- lib/kafka_rest/sender/payload/json_builder.rb
+- lib/kafka_rest/sender/kafka_sender.rb
+- lib/kafka_rest/sender/log_sender.rb
+- lib/kafka_rest/sender/test_sender.rb
 - lib/kafka_rest/version.rb
 - lib/kafka_rest/worker.rb
 - lib/kafka_rest/worker/consumer_manager.rb
@@ -196,24 +201,24 @@ homepage: https://github.com/konukhov/kafka-rest-rb
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: Kafka-REST proxy client for Ruby on Rails.
 test_files: []