kafka-rest-rb 0.1.0.alpha5 → 0.1.0.alpha6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27bd59add9955e9335735cb54c34d4bc66343f75
-  data.tar.gz: 24f6743e1828e96d689643e88308445fef656d39
+  metadata.gz: 2830f3a07c3fb219d42bf57f6339a7ecdc9bd471
+  data.tar.gz: cdfe237fd60fb23ec55ab65f237b61c70604c5e0
 SHA512:
-  metadata.gz: 725737a430db0a97edbf2649792a489a993ae99ad9a051b5a36e7300c0c32613e8c1f08837ebf4faee9e61127da2823047258b888d8215fd2b893c031dba4102
-  data.tar.gz: c0fa43a9dcb4591d24e33036a2cf923a8b9ce280e5dfa29af11530a1e44b99d90acd4733fea0f43c0715eb15bf43572d68748ec2a75ac5f100ed7309961693d4
+  metadata.gz: 8e345c4cc6b29a8fffe512c90502dc36db0c9f296c7859a11e1e76b63fc5dea3d3853f0ef016b4641f147399e443ad9ad7132e6985ea30c343f3d21a1ac7e962
+  data.tar.gz: 703f11ff97e5b580c67bd1a728f304cbc0bc42e65ba651b202b79e28aa5edd54b7c3eb96750df4d6ec83cc2aa8db2bd6ffd767d679e7b29a9609876afc2143da

data/lib/kafka_rest/client/middleware.rb

@@ -28,7 +28,11 @@ module KafkaRest
 
     class JsonResponse < FaradayMiddleware::ResponseMiddleware
       define_parser do |body|
-        MultiJson.load(body)
+        if body == ""
+          nil
+        else
+          MultiJson.load(body)
+        end
       end
     end
 

data/lib/kafka_rest/dsl.rb

@@ -5,7 +5,6 @@ module KafkaRest
 
     def option(name, opts = {})
       name         = name.to_s
-      required     = opts[:required] || false
       default      = opts[:default]
       validate     = opts[:validate] || ->(val) { true }
       error_msg    = opts[:error_message] || "`#{name}`'s value is invalid"
@@ -23,13 +22,7 @@ module KafkaRest
 
         class << self
           def get_#{name}
-            @#{name}.tap do |v|
-              if #{required} && v.nil?
-                raise KafkaRest::Dsl::MissingRequiredOption.new(
-                  "Missing required option `#{name}`"
-                )
-              end
-            end
+            @#{name}
           end
 
           def #{name}(val)

data/lib/kafka_rest/version.rb

@@ -1,3 +1,3 @@
 module KafkaRest
-  VERSION = '0.1.0.alpha5'
+  VERSION = '0.1.0.alpha6'
 end

data/lib/kafka_rest/worker.rb

@@ -12,6 +12,8 @@ module KafkaRest
     def initialize(client)
       @client = client
       @started = false
+      @util_threads = []
+      @out_pipe, @in_pipe = IO.pipe
       @thread_pool = Concurrent::ThreadPoolExecutor.new(
         min_threads: KafkaRest.config.worker_min_threads,
         max_threads: KafkaRest.config.worker_max_threads,
@@ -25,34 +27,85 @@ module KafkaRest
     end
 
     def start
+      trap('SIGINT') { @in_pipe.puts 'SIGINT' } # more signals
+
       begin
         @running = true
 
-        trap(:SIGINT) do
-          stop
-        end
+        util_thread { run_dead_cleaner }
+        util_thread { run_work_loop }
 
-        init_consumers
-        run_work_loop
+        wait_for_signal
       rescue => e
         logger.error "[Kafka REST] Got exception: #{e.class} (#{e.message})"
         e.backtrace.each { |msg| logger.error "\t #{msg}" }
         stop
+      ensure
+        [@in_pipe, @out_pipe].each &:close
       end
     end
 
     def stop
       logger.info "[Kafka REST] Stopping worker..."
+
       @running = false
-      remove_consumers
+      @util_threads.map &:join
+
+      logger.info "[Kafka REST] Bye."
+      exit(0)
     end
 
     private
 
-    def run_work_loop
+    # a dirty hack
+    def send_quit
+      @in_pipe.puts "SIGINT"
+    end
+
+    def wait_for_signal
+      while data = IO.select([@out_pipe])
+        signal = data.first[0].gets.strip
+        handle_signal(signal)
+      end
+    end
+
+    def handle_signal(signal)
+      case signal
+      when 'SIGINT'
+        stop
+      else
+        raise Interrupt
+      end
+    end
+
+    # Runs some job in an utility thread
+    def util_thread(&block)
+      parent = Thread.current
+
+      @util_threads << Thread.new(&block).tap do |t|
+        t.abort_on_exception = true
+      end
+    end
+
+    def run_dead_cleaner
       while @running
-        check_dead!
+        sleep(3)
+
+        dead = @consumers.select(&:dead?)
+
+        if @consumers.count == dead.count
+          logger.warn "All consumers are dead. Quitting..."
+          send_quit
+        else
+          dead.each(&:remove!)
+        end
+      end
+    end
 
+    def run_work_loop
+      init_consumers
+
+      while @running
         jobs = @consumers.select(&:poll?)
 
         if jobs.empty?
@@ -70,22 +123,16 @@ module KafkaRest
           sleep(BUSY_THREAD_POOL_DELAY)
         end
       end
-    end
 
-    def check_dead!
-      # Do we need this?
-      if @consumers.all?(&:dead?)
-        logger.info "[Kafka REST] All consumers are dead. Quitting..."
-        stop
-      end
+      remove_consumers
     end
 
     def init_consumers
-      @consumers.map &:add!
+      @consumers.select(&:initial?).each(&:add!)
     end
 
     def remove_consumers
-      @consumers.reject(&:initial?).map &:remove!
+      @consumers.reject(&:initial?).each(&:remove!)
     end
 
     def max_queue

data/lib/kafka_rest/worker/consumer.rb

@@ -0,0 +1,82 @@
+module KafkaRest
+  class Worker
+    class Consumer
+      # Does the actual work of working with kafka-rest api,
+      # processing messages and commiting offsets.
+      # This class is not thread-safe. In threaded environments
+      # it must be used within KafkaRest::Worker::ConsumerManager
+
+      include KafkaRest::Logging
+      extend Forwardable
+
+      def_delegators :@instance,
+                     :topic, :group_name, :poll_delay, :auto_commit,
+                     :offset_reset, :format, :max_bytes
+
+      def initialize(consumer, client)
+        @instance = consumer.new
+        @client = client
+        @id = nil
+        @uri = nil
+      end
+
+      def add!
+        params = {}
+        params[:auto_commit_enable] = auto_commit if auto_commit
+        params[:auto_offset_reset] = offset_reset if offset_reset
+        params[:format] = format if format
+
+        resp = @client.consumer_add(group_name, params)
+
+        @id  = resp.body['instance_id']
+        @uri = resp.body['base_uri']
+
+        logger.info "[Kafka REST] Added consumer #{@id}"
+      end
+
+      def remove!
+        resp = @client.consumer_remove(group_name, @id)
+        @id    = nil
+        @uri   = nil
+
+        logger.info "[Kafka REST] Removed consumer #{@id}"
+      end
+
+      def poll!
+        params = {}
+        params[:format] = format if format
+        params[:max_bytes] = max_bytes if max_bytes
+
+        logger.debug "[Kafka REST] Polling consumer #{@id} in group #{group_name}..."
+
+        resp = @client.consumer_consume_from_topic(group_name, @id, topic, params)
+        process_messages(resp.body)
+      end
+
+      def commit!
+        @client.consumer_commit_offsets(group_name, @id)
+      end
+
+      def added?
+        !@id.nil?
+      end
+
+      private
+
+      def process_messages(messages)
+        if messages.any?
+          messages.each do |msg|
+            logger.debug "[Kafka REST] Consumer #{@id} got message: #{msg}"
+            @instance.receive ConsumerMessage.new(msg, topic)
+          end
+
+          commit! unless auto_commit
+
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/kafka_rest/worker/consumer_manager.rb

@@ -1,45 +1,57 @@
 require 'concurrent/utility/monotonic_time'
+require 'kafka_rest/worker/consumer_message'
+require 'kafka_rest/worker/consumer'
 
 module KafkaRest
   class Worker
     class ConsumerManager
+      # Manages state and lifecycle of a Consumer instance
+
       STATES = [:initial, :idle, :working, :dead]
 
       include KafkaRest::Logging
 
       class << self
-        @@consumers = []
+        @@register_consumer_procs = []
+        @@consumers = nil
+
+        def register!(consumer)
+          # Delay consumers validating until they are actually needed
+          # and all classes are loaded.
+          @@register_consumer_procs << ->(consumers){
+            topic, group_name = consumer.get_topic, consumer.get_group_name
+
+            if topic.nil?
+              raise Exception.new("#{consumer.name}: topic must not be empty")
+            end
+
+            if group_name.nil?
+              raise Exception.new("#{consumer.name}: group_name must not be empty")
+            end
 
-        def register!(consumer_class)
-          # TODO: raise exception if group_id + topic are not unique
-          # TODO: Thread.current???
-          @@consumers << consumer_class
+            key = [topic, group_name]
+
+            if consumers.has_key?(key)
+              raise Exception.new("#{consumer.name}: group_name and topic are not unique")
+            end
+
+            { key => consumer }
+          }
         end
 
         def consumers
-          @@consumers
+          @@consumers ||= @@register_consumer_procs.reduce({}) do |c, p|
+            c.merge(p.call c)
+          end.values
         end
       end
 
-      extend Forwardable
-
-      def_delegators :@consumer,
-                     :topic,
-                     :group_name,
-                     :poll_delay,
-                     :auto_commit,
-                     :offset_reset,
-                     :format,
-                     :max_bytes
-
-      def initialize(client, consumer)
-        @client    = client
-        @consumer  = consumer.new
-        @id        = nil
-        @uri       = nil
-        @state     = :initial
-        @next_poll = Concurrent.monotonic_time
-        @lock      = Mutex.new
+      def initialize(client, klass)
+        @consumer   = Consumer.new(klass, client)
+        @state      = :initial
+        @poll_delay = klass.get_poll_delay
+        @next_poll  = Concurrent.monotonic_time
+        @lock       = Mutex.new
       end
 
       STATES.each do |state|
@@ -50,72 +62,52 @@ module KafkaRest
         }
       end
 
-      def poll?
-        with_lock {
+      def poll?(lock = true)
+        with_lock(lock) do
           idle?(false) && Concurrent.monotonic_time > @next_poll
-        }
+        end
       end
 
       def add!
-        params = {}.tap do |h|
-          auto_commit.nil? or h[:auto_commit_enable] = auto_commit
-          offset_reset and h[:auto_offset_reset] = offset_reset
-          format and h[:format] = format
+        with_lock do
+          return nil if @consumer.added?
+          @consumer.add!
+          @state = :idle
         end
-
-        resp   = @client.consumer_add(group_name, params)
-        @id    = resp.body['instance_id']
-        @uri   = resp.body['base_uri']
-        @state = :idle
-
-        logger.info "[Kafka REST] Added consumer #{@id}"
       end
 
       def remove!
-        resp = @client.consumer_remove(group_name, @id)
-        logger.info "[Kafka REST] Removed consumer #{@id}"
+        with_lock do
+          return nil unless @consumer.added?
+          @consumer.remove!
+          @state = :initial
+        end
       end
 
       def poll!
+        with_lock do
+          return nil unless idle?(false)
+          @state = :working
+        end
+
         begin
-          with_lock do
-            return false unless idle?(false)
-            @state = :working
+          unless @consumer.poll!
+            @next_poll = Concurrent.monotonic_time + @poll_delay
           end
 
-          logger.debug "Polling #{group_name}..."
-
-          params = {}.tap do |h|
-            format and h[:format] = format
-            max_bytes and h[:max_bytes] = max_bytes
+          with_lock { @state = :idle }
+        rescue Exception => e
+          # TODO:
+          # - Recover from Faraday errors.
+          # - Mark dead only when encountering application errors,
+          #   because it's obvious that after restart it will be raised again
+          logger.error "[Kafka REST] Consumer died due to an error"
+          logger.error "#{e.class}: #{e.message}"
+
+          e.backtrace.each do |s|
+            logger.error s
           end
 
-          messages = @client.consumer_consume_from_topic(
-            group_name,
-            @id,
-            topic,
-            params
-          ).body
-
-          if messages.any?
-            messages.each do |msg|
-              logger.debug "[Kafka REST] Consumer #{@id} got message: #{msg}"
-              @consumer.receive(msg)
-            end
-
-            unless auto_commit
-              @client.consumer_commit_offsets(group_name, @id)
-            end
-
-            with_lock { @state = :idle }
-          else
-            with_lock do
-              @next_poll = Concurrent.monotonic_time + poll_delay
-              @state = :idle
-            end
-          end
-        rescue Exception => e # TODO: handle errors
-          logger.warn "[Kafka REST] Consumer died due to error: #{e.class}, #{e.message}"
           with_lock { @state = :dead }
         end
       end

data/lib/kafka_rest/worker/consumer_message.rb

@@ -0,0 +1,16 @@
+module KafkaRest
+  class Worker
+    class ConsumerMessage
+      attr_reader :key, :value, :offset, :partition, :timestamp
+
+      def initialize(payload, topic)
+        @key = payload['key']
+        @value = payload['value']
+        @partition = payload['partition']
+        @timestamp = payload['timestamp']
+        @offset = payload['offset']
+        @topic = topic
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kafka-rest-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0.alpha5
+  version: 0.1.0.alpha6
 platform: ruby
 authors:
 - Theodore Konukhov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-26 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -195,7 +195,9 @@ files:
 - lib/kafka_rest/sender/test_sender.rb
 - lib/kafka_rest/version.rb
 - lib/kafka_rest/worker.rb
+- lib/kafka_rest/worker/consumer.rb
 - lib/kafka_rest/worker/consumer_manager.rb
+- lib/kafka_rest/worker/consumer_message.rb
 - perf/perftest
 homepage: https://github.com/konukhov/kafka-rest-rb
 licenses: