karafka 0.5.0 → 0.5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c40b95f3636ecc4ded9cbbfbb575129e5593560
-  data.tar.gz: 2ee0650b0244b772141af9566cfd7c41c0a8632d
+  metadata.gz: d7e097bb23271902edf811b27a20afb3da70aaf4
+  data.tar.gz: d104e55ea9c218e6d931d4922252dd5a2dc38c23
 SHA512:
-  metadata.gz: c850800bd7b372abe43df6d3179be95d69296b0e6369dc14bea3431f8fe76e3b33fbb6db360e0dd08218bacea183dfd31671c64ac6f3a122f897f2013653a765
-  data.tar.gz: 08f57ac78f4c00d0426d17a6a24044cf393f173a356b87865d605b7699b3333424159ae58f9f5ff8cf0bf0d540559d24d19abd69e76d3f5243253c66a25f56b1
+  metadata.gz: 8f010bd5993055cf89622d343aa09307539539b503ee42e722112ce7498b7cbc4c750db475518eec07bfe33d028f9befb1b2240b09ccfb44b90c786dae6f878e
+  data.tar.gz: 8feb1a74b19313ffb12c74152925e09d4307723bab316b46c5238ae7b8b9020cb3ab71f10886f78335cffcd6735e8a7aa4a882f17f583782aa2d055e49c4edef

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 # Karafka framework changelog
 
-## 0.5.0-beta
+## 0.5.0.1
+- Fixed inconsistency in responders non-required topic definition. Now only required: false available
+- #101
+- fix error on startup from waterdrop #102
+- Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
+- #105 - Karafka::Monitor#caller_label not working with inherited monitors
+- #99 - Standalone mode (without Sidekiq)
+- #97 - Buffer responders single topics before send (prevalidation)
+- Better control over consumer thanks to additional config options
+- #111 - Dynamic worker assignment based on the income params
+- Long shutdown time fix
+
+## 0.5.0
 - Removed Zookeeper totally as dependency
 - Better group and partition rebalancing
 - Automatic thread management (no need for tunning) - each topic is a separate actor/thread

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (0.5.0)
+    karafka (0.5.0.1)
       activesupport (~> 5.0)
       celluloid (~> 0.17)
       dry-configurable (~> 0.1.7)
@@ -166,10 +166,10 @@ GEM
     shoulda-context (1.2.1)
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
-    sidekiq (4.2.2)
+    sidekiq (4.2.3)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
-      rack-protection (~> 1.5)
+      rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
     simplecov (0.12.0)
       docile (~> 1.1.0)
@@ -192,9 +192,10 @@ GEM
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
-    waterdrop (0.3.2)
+    waterdrop (0.3.2.1)
       bundler
       connection_pool
+      dry-configurable (~> 0.1.7)
       null-logger
       rake
       ruby-kafka

data/README.md

@@ -31,6 +31,7 @@ Karafka not only handles incoming messages but also provides tools for building
         - [Parser](#parser)
         - [Interchanger](#interchanger)
         - [Responder](#responder)
+        - [Inline flag](#inline-flag)
     - [Receiving messages](#receiving-messages)
         - [Processing messages directly (without Sidekiq)](#processing-messages-directly-without-sidekiq)
     - [Sending messages from Karafka](#sending-messages-from-karafka)
@@ -39,6 +40,7 @@ Karafka not only handles incoming messages but also provides tools for building
   - [Important components](#important-components)
     - [Controllers](#controllers)
         - [Controllers callbacks](#controllers-callbacks)
+        - [Dynamic worker selection](#dynamic-worker-selection)
     - [Responders](#responders)
         - [Registering topics](#registering-topics)
         - [Responding on topics](#responding-on-topics)
@@ -77,7 +79,7 @@ In order to use Karafka framework, you need to have:
 
 ## Installation
 
-Karafka does not have a full installation shell command. In order to install it, please follow given steps:
+Karafka does not have a full installation shell command. In order to install it, please follow the below steps:
 
 Create a directory for your project:
 
@@ -91,7 +93,7 @@ Create a **Gemfile** with Karafka:
 ```ruby
 source 'https://rubygems.org'
 
-gem 'karafka', github: 'karafka/karafka'
+gem 'karafka'
 ```
 
 and run Karafka install CLI task:
@@ -105,13 +107,18 @@ bundle exec karafka install
 ### Application
 Karafka has following configuration options:
 
-| Option                 | Required | Value type        | Description                                                                                 |
-|------------------------|----------|-------------------|---------------------------------------------------------------------------------------------|
-| name                   | true     | String            | Application name                                                                            |
-| redis                  | true     | Hash              | Hash with Redis configuration options                                                       |
-| monitor                | false    | Object            | Monitor instance (defaults to Karafka::Monitor)                                             |
-| logger                 | false    | Object            | Logger instance (defaults to Karafka::Logger)                                               |
-| kafka.hosts            | false    | Array<String>     | Kafka server hosts. If 1 provided, Karafka will discover cluster structure automatically    |
+| Option                        | Required | Value type        | Description                                                                                                |
+|-------------------------------|----------|-------------------|------------------------------------------------------------------------------------------------------------|
+| name                          | true     | String            | Application name                                                                                           |
+| inline                        | false    | Boolean           | Do we want to perform logic without enqueuing it with Sidekiq (directly and asap)                          |
+| redis                         | true     | Hash              | Hash with Redis configuration options                                                                      |
+| monitor                       | false    | Object            | Monitor instance (defaults to Karafka::Monitor)                                                            |
+| logger                        | false    | Object            | Logger instance (defaults to Karafka::Logger)                                                              |
+| kafka.hosts                   | false    | Array<String>     | Kafka server hosts. If 1 provided, Karafka will discover cluster structure automatically                   |
+| kafka.session_timeout         | false    | Integer           | The number of seconds after which, if a consumer hasn't contacted the Kafka cluster, it will be kicked out |
+| kafka.offset_commit_interval  | false    | Integer           | The interval between offset commits in seconds                                                             |
+| kafka.offset_commit_threshold | false    | Integer           | The number of messages that can be processed before their offsets are committed                            |
+| kafka.heartbeat_interval      | false    | Integer           | The interval between heartbeats                                                                            |
 
 To apply this configuration, you need to use a *setup* method from the Karafka::App class (app.rb):
 
@@ -119,6 +126,7 @@ To apply this configuration, you need to use a *setup* method from the Karafka::
 class App < Karafka::App
   setup do |config|
     config.kafka.hosts = %w( 127.0.0.1:9092 )
+    config.inline = false
     config.redis = {
       url: 'redis://redis.example.com:7372/1'
     }
@@ -208,6 +216,7 @@ There are also several other methods available (optional):
   - *parser* - Class name - name of a parser class that we want to use to parse incoming data
   - *interchanger* - Class name - name of a interchanger class that we want to use to format data that we put/fetch into/from *#perform_async*
   - *responder* - Class name - name of a responder that we want to use to generate responses to other Kafka topics based on our processed data
+  - *inline* - Boolean - Do we want to perform logic without enqueuing it with Sidekiq (directly and asap) - overwrites global app setting
 
 ```ruby
 App.routes.draw do
@@ -218,6 +227,7 @@ App.routes.draw do
     parser Parsers::BinaryToJson
     interchanger Interchangers::Binary
     responder BinaryVideoProcessingResponder
+    inline true
   end
 
   topic :new_videos do
@@ -272,7 +282,7 @@ However, if you want to use a raw Sidekiq worker (without any Karafka additional
 ```ruby
 topic :incoming_messages do
   controller MessagesController
-  worker MyCustomController
+  worker MyCustomWorker
 end
 ```
 
@@ -289,7 +299,7 @@ Keep in mind, that params might be in two states: parsed or unparsed when passed
 
  - *parser* - Class name - name of a parser class that we want to use to parse incoming data
 
-Karafka by default will parse messages with a JSON parser. If you want to change this behaviour you need to set custom parser for each route. Parser needs to have a #parse method and raise error that is a ::Karafka::Errors::ParserError descendant when problem appears during parsing process.
+Karafka by default will parse messages with a JSON parser. If you want to change this behaviour you need to set a custom parser for each route. Parser needs to have a #parse method and raise an error that is a ::Karafka::Errors::ParserError descendant when problem appears during the parsing process.
 
 ```ruby
 class XmlParser
@@ -314,7 +324,7 @@ Note that parsing failure won't stop the application flow. Instead, Karafka will
 
 ##### Interchanger
 
- - *interchanger* - Class name - name of a interchanger class that we want to use to format data that we put/fetch into/from #perform_async.
+ - *interchanger* - Class name - name of an interchanger class that we want to use to format data that we put/fetch into/from #perform_async.
 
 Custom interchangers target issues with non-standard (binary, etc) data that we want to store when we do #perform_async. This data might be corrupted when fetched in a worker (see [this](https://github.com/karafka/karafka/issues/30) issue). With custom interchangers, you can encode/compress data before it is being passed to scheduling and decode/decompress it when it gets into the worker.
 
@@ -359,6 +369,17 @@ end
 
 For more details about responders, please go to the [using responders](#using-responders) section.
 
+##### Inline flag
+
+Inline flag allows you to disable Sidekiq usage by performing your #perform method business logic in the main Karafka server process.
+
+This flag be useful when you want to:
+
+  - process messages one by one in a single flow
+  - process messages as soon as possible (without Sidekiq delay)
+
+Note: Keep in mind, that by using this, you can significantly slow down Karafka. You also loose all the advantages of Sidekiq processing (reentrancy, retries, etc).
+
 ### Receiving messages
 
 Karafka framework has a long running server process that is responsible for receiving messages.
@@ -377,16 +398,24 @@ bundle exec karafka server --daemon
 
 #### Processing messages directly (without Sidekiq)
 
-If you don't want to use Sidekiq for processing and you would rather process messages directly in the main Karafka server process, you can do that using the *before_enqueue* callback inside of controller:
+If you don't want to use Sidekiq for processing and you would rather process messages directly in the main Karafka server process, you can do that by setting the *inline* flag either on an app level:
 
 ```ruby
-class UsersController < ApplicationController
-  before_enqueue :perform_directly
+class App < Karafka::App
+  setup do |config|
+    config.inline = false
+    # Rest of the config
+  end
+end
+```
 
-  # By throwing abort signal, Karafka will not schedule a background #perform task.
-  def perform_directly
-    User.create(params[:user])
-    throw(:abort)
+or per route (when you want to treat some routes in a different way):
+
+```ruby
+App.routes.draw do
+  topic :binary_video_details do
+    controller Videos::DetailsController
+    inline true
   end
 end
 ```
@@ -430,15 +459,15 @@ module Users
 end
 ```
 
-Appropriate responder will be used automatically when you invoke the **respond_with** controller method.
+The appropriate responder will be used automatically when you invoke the **respond_with** controller method.
 
-Why did we separate response layer from the controller layer? Because sometimes when you respond to multiple topics conditionally, that logic can be really complex and it is way better to manage and test it in isolation.
+Why did we separate the response layer from the controller layer? Because sometimes when you respond to multiple topics conditionally, that logic can be really complex and it is way better to manage and test it in isolation.
 
 For more details about responders DSL, please visit the [responders](#responders) section.
 
 #### Using WaterDrop directly
 
-It is not recommended (as it breaks responders validations and makes it harder to track data flow), but if you want to send messages outside of Karafka responders, you can to use **waterdrop** gem directly.
+It is not recommended (as it breaks responders validations and makes it harder to track data flow), but if you want to send messages outside of Karafka responders, you can to use the **waterdrop** gem directly.
 
 Example usage:
 
@@ -477,7 +506,7 @@ end
 
 #### Controllers callbacks
 
-You can add any number of *before_enqueue* callbacks. It can be method or block.
+You can add any number of *before_enqueue* callbacks. It can be a method or a block.
 before_enqueue acts in a similar way to Rails before_action so it should perform "lightweight" operations. You have access to params inside. Based on it you can define which data you want to receive and which not.
 
 **Warning**: keep in mind, that all *before_enqueue* blocks/methods are executed after messages are received. This is not executed in Sidekiq, but right after receiving the incoming message. This means, that if you perform "heavy duty" operations there, Karafka might significantly slow down.
@@ -519,6 +548,17 @@ Presented example controller will accept incoming messages from a Kafka topic na
 end
 ```
 
+#### Dynamic worker selection
+
+When you work with Karafka, you may want to schedule part of the jobs to a different worker based on the incoming params. This can be achieved by reassigning worker in the *#before_enqueue* block:
+
+```ruby
+before_enqueue do
+  self.worker = (params[:important] ? FastWorker : SlowWorker)
+end
+```
+
+
 ### Responders
 
 Responders are used to design and control response flow that comes from a single controller action. You might be familiar with a #respond_with Rails controller method. In Karafka it is an entrypoint to a responder *#respond*.
@@ -758,7 +798,7 @@ Want to use Karafka with Ruby on Rails or Sinatra? It can be done!
 Add Karafka to your Ruby on Rails application Gemfile:
 
 ```ruby
-gem 'karafka', github: 'karafka/karafka'
+gem 'karafka'
 ```
 
 Copy the **app.rb** file from your Karafka application into your Rails app (if you don't have this file, just create an empty Karafka app and copy it). This file is responsible for booting up Karafka framework. To make it work with Ruby on Rails, you need to load whole Rails application in this file. To do so, replace:
@@ -789,7 +829,7 @@ Sinatra applications differ from one another. There are single file applications
 Add Karafka to your Sinatra application Gemfile:
 
 ```ruby
-gem 'karafka', github: 'karafka/karafka'
+gem 'karafka'
 ```
 
 After that make sure that whole your application is loaded before setting up and booting Karafka (see Ruby on Rails integration for more details about that).
@@ -808,6 +848,7 @@ After that make sure that whole your application is loaded before setting up and
 
 ### Articles and references
 
+* [Karafka (Ruby + Kafka framework) 0.5.0 release details](http://dev.mensfeld.pl/2016/09/karafka-ruby-kafka-framework-0-5-0-release-details/)
 * [Karafka – Ruby micro-framework for building Apache Kafka message-based applications](http://dev.mensfeld.pl/2015/08/karafka-ruby-micro-framework-for-building-apache-kafka-message-based-applications/)
 * [Benchmarking Karafka – how does it handle multiple TCP connections](http://dev.mensfeld.pl/2015/11/benchmarking-karafka-how-does-it-handle-multiple-tcp-connections/)
 * [Karafka – Ruby framework for building Kafka message based applications (presentation)](http://mensfeld.github.io/karafka-framework-introduction/)

data/karafka.gemspec

@@ -7,11 +7,11 @@ Gem::Specification.new do |spec|
   spec.name          = 'karafka'
   spec.version       = ::Karafka::VERSION
   spec.platform      = Gem::Platform::RUBY
-  spec.authors       = ['Maciej Mensfeld', 'Pavlo Vavruk']
-  spec.email         = %w( maciej@mensfeld.pl pavlo.vavruk@gmail.com )
+  spec.authors       = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
+  spec.email         = %w( maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com )
   spec.homepage      = 'https://github.com/karafka/karafka'
-  spec.summary       = %q{ Ruby based Microframework for handling Apache Kafka incoming messages }
-  spec.description   = %q{ Microframework used to simplify Kafka based Ruby applications }
+  spec.summary       = %q{ Ruby based framework for working with Apache Kafka }
+  spec.description   = %q{ Framework used to simplify Apache Kafka based Ruby applications development }
   spec.license       = 'MIT'
 
   spec.add_development_dependency 'bundler', '~> 1.2'

data/lib/karafka.rb

@@ -23,6 +23,7 @@
   active_support/inflector
   karafka/loader
   karafka/status
+  karafka/routing/route
 ).each { |lib| require lib }
 
 # Karafka library

data/lib/karafka/base_controller.rb

@@ -64,7 +64,15 @@ module Karafka
 
     # This will be set based on routing settings
     # From 0.4 a single controller can handle multiple topics jobs
-    attr_accessor :group, :topic, :worker, :parser, :interchanger, :responder
+    # All the attributes are taken from route
+    Karafka::Routing::Route::ATTRIBUTES.each do |attr|
+      attr_reader attr
+
+      define_method(:"#{attr}=") do |new_attr_value|
+        instance_variable_set(:"@#{attr}", new_attr_value)
+        @params[attr] = new_attr_value if @params
+      end
+    end
 
     class << self
       # Creates a callback that will be executed before scheduling to Sidekiq
@@ -95,7 +103,7 @@ module Karafka
     # will schedule a perform task in sidekiq
     def schedule
       run_callbacks :schedule do
-        perform_async
+        inline ? perform_inline : perform_async
       end
     end
 
@@ -144,13 +152,20 @@ module Karafka
       responder.new.call(*data)
     end
 
+    # Executes perform code immediately (without enqueuing)
+    # @note Despite the fact, that workers won't be used, we still initialize all the
+    #   classes and other framework elements
+    def perform_inline
+      Karafka.monitor.notice(self.class, to_h)
+      perform
+    end
+
     # Enqueues the execution of perform method into a worker.
     # @note Each worker needs to have a class #perform_async method that will allow us to pass
     #   parameters into it. We always pass topic as a first argument and this request params
     #   as a second one (we pass topic to be able to build back the controller in the worker)
     def perform_async
       Karafka.monitor.notice(self.class, to_h)
-
       # We use @params directly (instead of #params) because of lazy loading logic that is behind
       # it. See Karafka::Params::Params class for more details about that
       worker.perform_async(

data/lib/karafka/base_responder.rb

@@ -16,10 +16,10 @@ module Karafka
   #     end
   #   end
   #
-  # @example Marking topic as optional (we won't have to use it)
+  # @example Marking topic as not required (we won't have to use it)
   #   class Responder < BaseResponder
   #     topic :required_topic
-  #     topic :new_action, optional: true
+  #     topic :new_action, required: false
   #
   #     def respond(data)
   #       respond_to :required_topic, data
@@ -51,6 +51,8 @@ module Karafka
     # Definitions of all topics that we want to be able to use in this responder should go here
     class_attribute :topics
 
+    attr_reader :messages_buffer
+
     class << self
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
@@ -65,7 +67,7 @@ module Karafka
     # Creates a responder object
     # @return [Karafka::BaseResponder] base responder descendant responder
     def initialize
-      @used_topics = []
+      @messages_buffer = {}
     end
 
     # Performs respond and validates that all the response requirement were met
@@ -76,6 +78,7 @@ module Karafka
     def call(*data)
       respond(*data)
       validate!
+      deliver!
     end
 
     private
@@ -97,22 +100,30 @@ module Karafka
     def respond_to(topic, data)
       Karafka.monitor.notice(self.class, topic: topic, data: data)
 
-      topic = topic.to_s
-      @used_topics << topic
-
-      ::WaterDrop::Message.new(
-        topic,
-        data.is_a?(String) ? data : data.to_json
-      ).send!
+      messages_buffer[topic.to_s] ||= []
+      messages_buffer[topic.to_s] << (data.is_a?(String) ? data : data.to_json)
     end
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
     def validate!
+      used_topics = messages_buffer.map do |key, data_elements|
+        Array.new(data_elements.count) { key }
+      end
+
       Responders::UsageValidator.new(
         self.class.topics || {},
-        @used_topics
+        used_topics.flatten
       ).validate!
     end
+
+    # Takes all the messages from the buffer and delivers them one by one
+    # @note This method is executed after the validation, so we're sure that
+    #   what we send is legit and it will go to a proper topics
+    def deliver!
+      messages_buffer.each do |topic, data_elements|
+        data_elements.each { |data| ::WaterDrop::Message.new(topic, data).send! }
+      end
+    end
   end
 end

data/lib/karafka/cli/info.rb

@@ -12,6 +12,7 @@ module Karafka
         info = [
           "Karafka framework version: #{Karafka::VERSION}",
           "Application name: #{config.name}",
+          "Inline mode: #{config.inline}",
           "Number of threads: #{config.concurrency}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}",

data/lib/karafka/cli/routes.rb

@@ -10,12 +10,9 @@ module Karafka
       def call
         routes.each do |route|
           puts "#{route.topic}:"
-          print('Group', route.group)
-          print('Controller', route.controller)
-          print('Worker', route.worker)
-          print('Parser', route.parser)
-          print('Interchanger', route.interchanger)
-          print('Responder', route.responder)
+          Karafka::Routing::Route::ATTRIBUTES.each do |attr|
+            print(attr.to_s.capitalize, route.public_send(attr))
+          end
         end
       end
 

data/lib/karafka/connection/topic_consumer.rb

@@ -22,7 +22,7 @@ module Karafka
 
       # Gracefuly stops topic consumption
       def stop
-        kafka_consumer.stop
+        @kafka_consumer&.stop
         @kafka_consumer = nil
       end
 
@@ -39,7 +39,14 @@ module Karafka
           client_id: ::Karafka::App.config.name
         )
 
-        @kafka_consumer = kafka.consumer(group_id: @route.group)
+        @kafka_consumer = kafka.consumer(
+          group_id: @route.group,
+          session_timeout: ::Karafka::App.config.kafka.session_timeout,
+          offset_commit_interval: ::Karafka::App.config.kafka.offset_commit_interval,
+          offset_commit_threshold: ::Karafka::App.config.kafka.offset_commit_threshold,
+          heartbeat_interval: ::Karafka::App.config.kafka.heartbeat_interval
+        )
+
         @kafka_consumer.subscribe(@route.topic)
         @kafka_consumer
       end

data/lib/karafka/monitor.rb

@@ -71,7 +71,10 @@ module Karafka
     # @example Check label of method that invoked #notice_error
     #   caller_label #=> 'rescue in target'
     def caller_label
-      caller_locations(1, 2)[1].label
+      # We need to calculate ancestors because if someone inherits
+      # from this  class, caller chains is longer
+      index = self.class.ancestors.index(Karafka::Monitor) + 1
+      caller_locations(index, 2)[1].label
     end
 
     # @return [Logger] logger instance

data/lib/karafka/params/params.rb

@@ -50,7 +50,8 @@ module Karafka
             controller: controller.class,
             worker: controller.worker,
             parser: controller.parser,
-            topic: controller.topic
+            topic: controller.topic,
+            responder: controller.responder
           )
         end
       end

data/lib/karafka/responders/topic.rb

@@ -4,7 +4,7 @@ module Karafka
     # @example Define topic (required by default)
     #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
     # @example Define optional topic
-    #   Karafka::Responders::Topic.new(:topic_name, optional: true)
+    #   Karafka::Responders::Topic.new(:topic_name, required: false)
     # @example Define topic that on which we want to respond multiple times
     #   Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
     class Topic
@@ -22,8 +22,7 @@ module Karafka
 
       # @return [Boolean] is this a required topic (if not, it is optional)
       def required?
-        return false if @options[:optional]
-        @options[:required] || true
+        @options.key?(:required) ? @options[:required] : true
       end
 
       # @return [Boolean] do we expect to use it multiple times in a single respond flow

data/lib/karafka/responders/usage_validator.rb

@@ -40,6 +40,7 @@ module Karafka
       def validate_usage_of!(used_topic)
         raise(Errors::UnregisteredTopic, used_topic) unless @registered_topics[used_topic]
         return if @registered_topics[used_topic].multiple_usage?
+        return unless @registered_topics[used_topic].required?
         return if @used_topics.count(used_topic) < 2
         raise(Errors::TopicMultipleUsage, used_topic)
       end

data/lib/karafka/routing/builder.rb

@@ -18,6 +18,7 @@ module Karafka
         parser
         interchanger
         responder
+        inline
       ).freeze
 
       # All those options should be set on the route level

data/lib/karafka/routing/route.rb

@@ -13,7 +13,17 @@ module Karafka
       NAME_FORMAT = /\A(\w|\-)+\z/
 
       # Options that we can set per each route
-      attr_writer :group, :topic, :worker, :parser, :interchanger, :responder
+      ATTRIBUTES = %i(
+        group
+        topic
+        worker
+        parser
+        interchanger
+        responder
+        inline
+      ).freeze
+
+      ATTRIBUTES.each { |attr| attr_writer(attr) }
 
       # This we can get "directly" because it does not have any details, etc
       attr_accessor :controller
@@ -23,11 +33,7 @@ module Karafka
       # everywhere except Karafka server command, those would not be initialized on time - for
       # example for Sidekiq
       def build
-        group
-        worker
-        parser
-        interchanger
-        responder
+        ATTRIBUTES.each { |attr| send(attr) }
         self
       end
 
@@ -68,6 +74,14 @@ module Karafka
         @interchanger ||= Karafka::Params::Interchanger
       end
 
+      # @return [Boolean] Should we perform execution in the background (default) or
+      #   inline. This can be set globally and overwritten by a per route setting
+      # @note This method can be set to false, so direct assigment ||= would not work
+      def inline
+        return @inline unless @inline.nil?
+        @inline = Karafka::App.config.inline
+      end
+
       # Checks if topic and group have proper format (acceptable by Kafka)
       # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
       # @raise [Karafka::Errors::InvalidGroupName] raised when group name is invalid

data/lib/karafka/routing/router.rb

@@ -14,14 +14,11 @@ module Karafka
       # Builds a controller instance that should handle message from a given topic
       # @return [Karafka::BaseController] base controller descendant instance object
       def build
-        controller = route.controller.new
-        controller.topic = route.topic
-        controller.parser = route.parser
-        controller.worker = route.worker
-        controller.interchanger = route.interchanger
-        controller.responder = route.responder
-
-        controller
+        route.controller.new.tap do |ctrl|
+          Karafka::Routing::Route::ATTRIBUTES.each do |attr|
+            ctrl.public_send(:"#{attr}=", route.public_send(attr))
+          end
+        end
       end
 
       private

data/lib/karafka/server.rb

@@ -25,7 +25,7 @@ module Karafka
       def bind_on_sigint
         process.on_sigint do
           Karafka::App.stop!
-          consumers.map(&:stop) if Karafka::App.running?
+          consumers.map(&:stop)
           exit
         end
       end
@@ -34,7 +34,7 @@ module Karafka
       def bind_on_sigquit
         process.on_sigquit do
           Karafka::App.stop!
-          consumers.map(&:stop) if Karafka::App.running?
+          consumers.map(&:stop)
           exit
         end
       end

data/lib/karafka/setup/config.rb

@@ -15,6 +15,8 @@ module Karafka
       # Available settings
       # option name [String] current app name - used to provide default Kafka groups namespaces
       setting :name
+      # If inline is set to true, we won't enqueue jobs, instead we will run them immediately
+      setting :inline, false
       # option logger [Instance] logger that we want to use
       setting :logger, ::Karafka::Logger.instance
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
@@ -25,7 +27,21 @@ module Karafka
       setting :redis
       # option kafka [Hash] - optional - kafka configuration options (hosts)
       setting :kafka do
+        # Array with at least one host
         setting :hosts
+        # option session_timeout [Integer] the number of seconds after which, if a client
+        #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
+        setting :session_timeout, 30
+        # option offset_commit_interval [Integer] the interval between offset commits,
+        #   in seconds.
+        setting :offset_commit_interval, 10
+        # option offset_commit_threshold [Integer] the number of messages that can be
+        #   processed before their offsets are committed. If zero, offset commits are
+        #   not triggered by message processing.
+        setting :offset_commit_threshold, 0
+        # option heartbeat_interval [Integer] the interval between heartbeats; must be less
+        #   than the session window.
+        setting :heartbeat_interval, 10
       end
 
       # This is configured automatically, don't overwrite it!

data/lib/karafka/setup/configurators/water_drop.rb

@@ -9,7 +9,7 @@ module Karafka
             water_config.send_messages = true
             water_config.connection_pool_size = config.concurrency
             water_config.connection_pool_timeout = 1
-            water_config.kafka_hosts = config.kafka.hosts
+            water_config.kafka.hosts = config.kafka.hosts
             water_config.raise_on_failure = true
           end
         end

data/lib/karafka/templates/app.rb.example

@@ -8,11 +8,12 @@ Karafka::Loader.new.load(Karafka::App.root)
 # App class
 class App < Karafka::App
   setup do |config|
-    config.kafka = { hosts: %w( 127.0.0.1:9092 ) }
+    config.kafka.hosts = %w( 127.0.0.1:9092 )
     config.name = 'example_app'
     config.redis = {
       url: 'redis://localhost:6379'
     }
+    config.inline = false
   end
 
   routes.draw do

data/lib/karafka/version.rb

@@ -2,5 +2,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '0.5.0'
+  VERSION = '0.5.0.1'
 end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.0.1
 platform: ruby
 authors:
 - Maciej Mensfeld
 - Pavlo Vavruk
+- Adam Gwozdowski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-30 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -165,10 +166,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.7
-description: " Microframework used to simplify Kafka based Ruby applications "
+description: " Framework used to simplify Apache Kafka based Ruby applications development "
 email:
 - maciej@mensfeld.pl
 - pavlo.vavruk@gmail.com
+- adam99g@gmail.com
 executables:
 - karafka
 extensions: []
@@ -263,5 +265,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Ruby based Microframework for handling Apache Kafka incoming messages
+summary: Ruby based framework for working with Apache Kafka
 test_files: []