karafka 0.6.0.rc2 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1ac31c4f943edba6e25886bc34b2bfad35e69fb
-  data.tar.gz: 488973afae1eb933b27dec7dd2fdcbb21fe6edae
+  metadata.gz: dff71b9105c2f08a83acaf4938b061935ccebba7
+  data.tar.gz: 7cc659012bbffedeb63e7195d21dafb780118dbf
 SHA512:
-  metadata.gz: 7e0ce78c37d1309d7b6adc51b4c59545df6eed3ee1dca0441c751b309090bd4d62e9da100fbe8fb7086af1b4f1d600ead0f9f7b873bf3d45e8f95092c30cdd76
-  data.tar.gz: d2555a9f9ca030df2d31a49d2a272e4d13b681ad53590708c6ed960004690761a0e3cf9e88ee000a0870d48d60e8987a7c0bf619c10cbfc5dad057fb374115b3
+  metadata.gz: 3bf047a983f77817ea41240455e4ec8a00d342ee786a1df5770a2c1d80ca8fe5c366308368e4fe6f537b75c54f5512fb5afdac074719ad141ea0495b8247c5ef
+  data.tar.gz: '09020aca3024ed6abee2e63466527acbf4c08d0aff3ef80d4a86169bc14b2afb373d06bcbd1035905959d732af2928abe3281fb9c10808c201bf3d0f204134db'

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka framework changelog
 
-## 0.6.0.rc2 - released
+## 1.0.0.rc1
 
 ### Closed issues:
 
@@ -34,6 +34,10 @@
 - (optional) pausing upon processing failures ```pause_timeout```
 - Karafka console main process no longer intercepts irb errors
 - Wiki updates
+- #204 - Long running controllers
+- Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
+- #207 - Rename before_enqueued to after_received
+- #147 - Deattach Karafka from Sidekiq by extracting Sidekiq backend
 
 ### New features and improvements
 
@@ -44,6 +48,7 @@
 - Introduced the ```#batch_processing``` config flag (config for #126) that can be set per each consumer_group
 - Added support for partition, offset and partition key in the params hash
 - ```name``` option in config renamed to ```client_id```
+- Long running controllers with ```persistent``` flag on a topic config level, to make controller instances persistent between messages batches (single controller instance per topic per partition no per messages batch) - turned on by default
 
 ### Incompatibilities
 
@@ -57,7 +62,10 @@
 - Renamed content to value to better resemble ruby-kafka internal messages naming convention
 - When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
 - Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
-- Renamed ```inline_processing``` to ```processing_backend``` to reach 1.0 future compatibility
+- Renamed ```inline_processing``` to ```backend``` to reach 1.0 future compatibility
+- Single controller **needs** to be used for a single topic consumption
+- Renamed ```before_enqueue``` to ```after_received``` to better resemble internal logic, since for inline backend, there is no enqueue.
+- Due to the level on which topic and controller are related (class level), the dynamic worker selection is no longer available.
 
 ### Other changes
 - PolishGeeksDevTools removed (in favour of Coditsu)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (0.6.0.rc2)
+    karafka (1.0.0.rc1)
       activesupport (>= 5.0)
       celluloid
       dry-configurable (~> 0.7)
@@ -11,7 +11,6 @@ PATH
       rake (>= 11.3)
       require_all (>= 1.4)
       ruby-kafka (>= 0.4)
-      sidekiq (>= 4.2)
       thor (~> 0.19)
       waterdrop (>= 0.4)
 
@@ -49,7 +48,7 @@ GEM
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.3.1)
+    dry-core (0.3.3)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
     dry-logic (0.4.1)
@@ -71,11 +70,8 @@ GEM
       dry-equalizer (~> 0.2)
       dry-logic (~> 0.4, >= 0.4.0)
       dry-types (~> 0.11.0)
-    envlogic (1.0.3)
+    envlogic (1.0.4)
       activesupport
-    ffi (1.9.18)
-    gssapi (1.2.0)
-      ffi (>= 1.0.1)
     hitimes (1.2.6)
     i18n (0.8.6)
     inflecto (0.0.2)
@@ -83,11 +79,7 @@ GEM
     minitest (5.10.3)
     multi_json (1.12.1)
     null-logger (0.1.4)
-    rack (2.0.3)
-    rack-protection (2.0.0)
-      rack
     rake (12.0.0)
-    redis (3.3.3)
     require_all (1.4.0)
     rspec (3.6.0)
       rspec-core (~> 3.6.0)
@@ -102,13 +94,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.6.0)
     rspec-support (3.6.0)
-    ruby-kafka (0.4.0)
-      gssapi (>= 1.2.0)
-    sidekiq (5.0.4)
-      concurrent-ruby (~> 1.0)
-      connection_pool (~> 2.2, >= 2.2.0)
-      rack-protection (>= 1.5.0)
-      redis (~> 3.3, >= 3.3.3)
+    ruby-kafka (0.4.1)
     simplecov (0.15.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)

data/karafka.gemspec

@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
   spec.license     = 'MIT'
 
   spec.add_dependency 'ruby-kafka', '>= 0.4'
-  spec.add_dependency 'sidekiq', '>= 4.2'
   spec.add_dependency 'celluloid'
   spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'waterdrop', '>= 0.4'

data/lib/karafka.rb

@@ -1,18 +1,11 @@
 # frozen_string_literal: true
 
 %w[
-  rake
-  ostruct
-  rubygems
-  bundler
   English
+  bundler
   celluloid/current
   waterdrop
-  pathname
-  timeout
-  logger
   kafka
-  sidekiq
   envlogic
   thor
   fileutils
@@ -26,9 +19,6 @@
   active_support/descendants_tracker
   active_support/inflector
   karafka/loader
-  karafka/setup/config
-  karafka/status
-  karafka/routing/router
 ].each(&method(:require))
 
 # Karafka library
@@ -64,7 +54,7 @@ module Karafka
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
-    #   It will be used for console/workers/etc
+    #   It will be used for console/controllers/etc
     # @example Standard only-Karafka case
     #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
     # @example Non standard case

data/lib/karafka/attributes_map.rb

@@ -33,13 +33,12 @@ module Karafka
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
         (config_adapter[:subscription] + %i[
-          processing_backend
+          backend
           name
-          worker
           parser
-          interchanger
           responder
           batch_processing
+          persistent
         ]).uniq
       end
 

data/lib/karafka/backends/inline.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all different backends Karafka supports
+  module Backends
+    # Backend that just runs stuff asap without any scheduling
+    module Inline
+      private
+
+      # Executes perform code immediately (without enqueuing)
+      def process
+        Karafka.monitor.notice(self.class, params_batch)
+        perform
+      end
+    end
+  end
+end

data/lib/karafka/base_controller.rb

@@ -3,11 +3,11 @@
 # Karafka module namespace
 module Karafka
   # Base controller from which all Karafka controllers should inherit
-  # Similar to Rails controllers we can define before_enqueue callbacks
+  # Similar to Rails controllers we can define after_received callbacks
   # that will be executed
   #
-  # Note that if before_enqueue return false, the chain will be stopped and
-  #   the perform method won't be executed in sidekiq (won't peform_async it)
+  # Note that if after_received return false, the chain will be stopped and
+  #   the perform method won't be executed
   #
   # @example Create simple controller
   #   class ExamplesController < Karafka::BaseController
@@ -16,9 +16,9 @@ module Karafka
   #     end
   #   end
   #
-  # @example Create a controller with a block before_enqueue
+  # @example Create a controller with a block after_received
   #   class ExampleController < Karafka::BaseController
-  #     before_enqueue do
+  #     after_received do
   #       # Here we should have some checking logic
   #       # If false is returned, won't schedule a perform action
   #     end
@@ -28,9 +28,9 @@ module Karafka
   #     end
   #   end
   #
-  # @example Create a controller with a method before_enqueue
+  # @example Create a controller with a method after_received
   #   class ExampleController < Karafka::BaseController
-  #     before_enqueue :before_method
+  #     after_received :after_received_method
   #
   #     def perform
   #       # some logic here
@@ -38,90 +38,72 @@ module Karafka
   #
   #     private
   #
-  #     def before_method
+  #     def after_received_method
   #       # Here we should have some checking logic
   #       # If false is returned, won't schedule a perform action
   #     end
   #   end
-  #
-  # @example Create a controller with an after_failure action
-  #   class ExampleController < Karafka::BaseController
-  #     def perform
-  #       # some logic here
-  #     end
-  #
-  #     def after_failure
-  #       # action taken in case perform fails
-  #     end
-  #   end
   class BaseController
     extend ActiveSupport::DescendantsTracker
     include ActiveSupport::Callbacks
 
-    # The schedule method is wrapped with a set of callbacks
+    # The call method is wrapped with a set of callbacks
     # We won't run perform at the backend if any of the callbacks
     # returns false
     # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
-    define_callbacks :schedule
+    define_callbacks :after_received
 
-    # Each controller instance is always bind to a single topic. We don't place it on a class
-    # level because some programmers use same controller for multiple topics
-    attr_accessor :topic
     attr_accessor :params_batch
 
     class << self
-      # Creates a callback that will be executed before scheduling to Sidekiq
+      attr_reader :topic
+
+      # Assigns a topic to a controller and build up proper controller functionalities, so it can
+      #   cooperate with the topic settings
+      # @param topic [Karafka::Routing::Topic]
+      # @return [Karafka::Routing::Topic] assigned topic
+      def topic=(topic)
+        @topic = topic
+        Controllers::Includer.call(self)
+        @topic
+      end
+
+      # Creates a callback that will be executed after receiving message but before executing the
+      #   backend for processing
       # @param method_name [Symbol, String] method name or nil if we plan to provide a block
       # @yield A block with a code that should be executed before scheduling
-      # @note If value returned is false, will chalt the chain and not schedlue to Sidekiq
-      # @example Define a block before_enqueue callback
-      #   before_enqueue do
+      # @example Define a block after_received callback
+      #   after_received do
       #     # logic here
       #   end
       #
-      # @example Define a class name before_enqueue callback
-      #   before_enqueue :method_name
-      def before_enqueue(method_name = nil, &block)
-        set_callback :schedule, :before, method_name ? method_name : block
+      # @example Define a class name after_received callback
+      #   after_received :method_name
+      def after_received(method_name = nil, &block)
+        set_callback :after_received, :before, method_name ? method_name : block
       end
     end
 
+    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
+    def topic
+      self.class.topic
+    end
+
     # Creates lazy loaded params batch object
     # @note Until first params usage, it won't parse data at all
     # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from Sidekiq, etc)
+    #   content (from Kafka) or messages inside a hash (from backend, etc)
     # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
     def params_batch=(messages)
       @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
     end
 
-    # @return [Karafka::Params::Params] params instance for non batch processed controllers
-    # @raise [Karafka::Errors::ParamsMethodUnavailable] raised when we try to use params
-    #   method in a batch_processed controller
-    def params
-      raise Karafka::Errors::ParamsMethodUnavailable if topic.batch_processing
-      params_batch.first
-    end
-
     # Executes the default controller flow, runs callbacks and if not halted
-    # will schedule a call task in sidekiq
-    def schedule
-      run_callbacks :schedule do
-        case topic.processing_backend
-        when :inline then
-          call_inline
-        when :sidekiq then
-          call_async
-        else
-          raise Errors::InvalidProcessingBackend, topic.processing_backend
-        end
-      end
-    end
-
-    # @note We want to leave the #perform method as a public API, but just in case we will do some
-    #   pre or post processing we use call method instead of directly executing #perform
+    # will call process method of a proper backend
     def call
-      perform
+      run_callbacks :after_received do
+        process
+      end
     end
 
     private
@@ -132,43 +114,5 @@ module Karafka
     def perform
       raise NotImplementedError, 'Implement this in a subclass'
     end
-
-    # @return [Karafka::BaseResponder] responder instance if defined
-    # @return [nil] nil if no responder for this controller
-    def responder
-      @responder ||= topic.responder&.new(topic.parser)
-    end
-
-    # Responds with given data using given responder. This allows us to have a similar way of
-    # defining flows like synchronous protocols
-    # @param data Anything we want to pass to responder based on which we want to trigger further
-    #   Kafka responding
-    # @raise [Karafka::Errors::ResponderMissing] raised when we don't have a responder defined,
-    #   but we still try to use this method
-    def respond_with(*data)
-      raise(Errors::ResponderMissing, self.class) unless responder
-      Karafka.monitor.notice(self.class, data: data)
-      responder.call(*data)
-    end
-
-    # Executes perform code immediately (without enqueuing)
-    # @note Despite the fact, that workers won't be used, we still initialize all the
-    #   classes and other framework elements
-    def call_inline
-      Karafka.monitor.notice(self.class, params_batch)
-      call
-    end
-
-    # Enqueues the execution of perform method into a worker.
-    # @note Each worker needs to have a class #perform_async method that will allow us to pass
-    #   parameters into it. We always pass topic as a first argument and this request params_batch
-    #   as a second one (we pass topic to be able to build back the controller in the worker)
-    def call_async
-      Karafka.monitor.notice(self.class, params_batch)
-      topic.worker.perform_async(
-        topic.id,
-        topic.interchanger.load(params_batch.to_a)
-      )
-    end
   end
 end

data/lib/karafka/base_responder.rb

@@ -138,6 +138,25 @@ module Karafka
       raise Karafka::Errors::InvalidResponderUsage, result.errors
     end
 
+    # Takes all the messages from the buffer and delivers them one by one
+    # @note This method is executed after the validation, so we're sure that
+    #   what we send is legit and it will go to a proper topics
+    def deliver!
+      messages_buffer.each do |topic, data_elements|
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
+
+        data_elements.each do |(data, options)|
+          ::WaterDrop::Message.new(
+            mapped_topic,
+            data,
+            options
+          ).send!
+        end
+      end
+    end
+
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
@@ -157,24 +176,5 @@ module Karafka
       messages_buffer[topic.to_s] ||= []
       messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
     end
-
-    # Takes all the messages from the buffer and delivers them one by one
-    # @note This method is executed after the validation, so we're sure that
-    #   what we send is legit and it will go to a proper topics
-    def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
-        data_elements.each do |(data, options)|
-          ::WaterDrop::Message.new(
-            mapped_topic,
-            data,
-            options
-          ).send!
-        end
-      end
-    end
   end
 end