karafka 1.0.0 → 1.2.0

data/bin/karafka

@@ -1,7 +1,19 @@
 #!/usr/bin/env ruby
 
 require 'karafka'
-require Karafka.boot_file.to_s
+
+# If there is a boot file, we need to require it as we expect it to contain
+# Karafka app setup, routes, etc
+if File.exist?(Karafka.boot_file)
+  require Karafka.boot_file.to_s
+else
+  # However when it is unavailable, we still want to be able to run help command
+  # and install command as they don't require configured app itself to run
+  raise(
+    Karafka::Errors::MissingBootFile,
+    Karafka.boot_file
+  ) unless %w[-h install].include?(ARGV[0])
+end
 
 Karafka::Cli.prepare
 Karafka::Cli.start

data/config/errors.yml

@@ -0,0 +1,6 @@
+en:
+  errors:
+    broker_schema?: >
+      has an invalid format.
+      Expected schema, host and port number.
+      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092

data/karafka.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
@@ -16,17 +16,18 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'ruby-kafka', '>= 0.4'
-  spec.add_dependency 'celluloid'
-  spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'waterdrop', '>= 0.4'
-  spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'activesupport', '>= 5.0'
-  spec.add_dependency 'dry-validation', '~> 0.11'
+  spec.add_dependency 'activesupport', '>= 4.0'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-inflector', '~> 0.1.1'
+  spec.add_dependency 'dry-monitor', '~> 0.1'
+  spec.add_dependency 'dry-validation', '~> 0.11'
+  spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
+  spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'require_all', '>= 1.4'
+  spec.add_dependency 'ruby-kafka', '>= 0.5.3'
+  spec.add_dependency 'thor', '~> 0.19'
+  spec.add_dependency 'waterdrop', '~> 1.2'
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka.rb

@@ -2,8 +2,6 @@
 
 %w[
   English
-  bundler
-  celluloid/current
   waterdrop
   kafka
   envlogic
@@ -13,11 +11,9 @@
   require_all
   dry-configurable
   dry-validation
+  dry/inflector
+  dry/monitor/notifications
   active_support/callbacks
-  active_support/core_ext/class/subclasses
-  active_support/core_ext/hash/indifferent_access
-  active_support/descendants_tracker
-  active_support/inflector
   karafka/loader
 ].each(&method(:require))
 
@@ -31,14 +27,14 @@ module Karafka
       @logger ||= App.config.logger
     end
 
-    # @return [::Karafka::Monitor] monitor that we want to use. Will use dummy monitor by default
+    # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
     end
 
     # @return [String] root path of this gem
     def gem_root
-      Pathname.new(File.expand_path('../..', __FILE__))
+      Pathname.new(File.expand_path('..', __dir__))
     end
 
     # @return [String] Karafka app root path (user application path)
@@ -48,13 +44,13 @@ module Karafka
 
     # @return [String] path to Karafka gem root core
     def core_root
-      Pathname.new(File.expand_path('../karafka', __FILE__))
+      Pathname.new(File.expand_path('karafka', __dir__))
     end
 
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
-    #   It will be used for console/controllers/etc
+    #   It will be used for console/consumers/etc
     # @example Standard only-Karafka case
     #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
     # @example Non standard case
@@ -66,4 +62,17 @@ module Karafka
   end
 end
 
+%w[
+  callbacks
+  callbacks/*
+  setup/dsl
+  setup/config
+  status
+  schemas/config
+  schemas/consumer_group_topic
+  schemas/consumer_group
+].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
+
 Karafka::Loader.load!(Karafka.core_root)
+Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
+Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/app.rb

@@ -3,14 +3,10 @@
 module Karafka
   # App class
   class App
-    class << self
-      # Sets up the whole configuration
-      # @param [Block] block configuration block
-      def setup(&block)
-        Setup::Config.setup(&block)
-        initialize!
-      end
+    extend Setup::Dsl
+    extend Callbacks::Dsl
 
+    class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
@@ -19,11 +15,7 @@ module Karafka
       def boot!
         Setup::Config.validate!
         Setup::Config.setup_components
-      end
-
-      # @return [Karafka::Config] config instance
-      def config
-        Setup::Config.config
+        Callbacks.after_init(Karafka::App.config)
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
@@ -33,7 +25,7 @@ module Karafka
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.public_send(delegated)
+          Status.instance.send(delegated)
         end
       end
 
@@ -41,10 +33,11 @@ module Karafka
       %i[
         root
         env
-        logger monitor
+        logger
+        monitor
       ].each do |delegated|
         define_method(delegated) do
-          Karafka.public_send(delegated)
+          Karafka.send(delegated)
         end
       end
     end

data/lib/karafka/attributes_map.rb

@@ -21,12 +21,12 @@ module Karafka
             offset_retention_time heartbeat_interval
           ],
           subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_wait_time],
+          consuming: %i[min_bytes max_bytes max_wait_time],
           pausing: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
-          ignored: %i[reconnect_timeout]
+          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
         }
       end
 
@@ -37,7 +37,7 @@ module Karafka
           name
           parser
           responder
-          batch_processing
+          batch_consuming
           persistent
         ]).uniq
       end
@@ -52,7 +52,7 @@ module Karafka
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
         ignored_settings = config_adapter[:subscription]
         defined_settings = config_adapter.values.flatten
-        karafka_settings = %i[batch_consuming]
+        karafka_settings = %i[batch_fetching]
         # This is a drity and bad hack of dry-configurable to get keys before setting values
         dynamically_proxied = Karafka::Setup::Config
                               ._settings

data/lib/karafka/backends/inline.rb

@@ -7,10 +7,9 @@ module Karafka
     module Inline
       private
 
-      # Executes perform code immediately (without enqueuing)
+      # Executes consume code immediately (without enqueuing)
       def process
-        Karafka.monitor.notice(self.class, params_batch)
-        perform
+        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
       end
     end
   end

data/lib/karafka/base_consumer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Karafka module namespace
+module Karafka
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
+    extend ActiveSupport::DescendantsTracker
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    def_delegator :client, :mark_as_consumed
+
+    private :mark_as_consumed
+
+    class << self
+      attr_reader :topic
+
+      # Assigns a topic to a consumer and builds up proper consumer functionalities
+      #   so that it can cooperate with the topic settings
+      # @param topic [Karafka::Routing::Topic]
+      # @return [Karafka::Routing::Topic] assigned topic
+      def topic=(topic)
+        @topic = topic
+        Consumers::Includer.call(self)
+      end
+    end
+
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    def topic
+      self.class.topic
+    end
+
+    # Creates lazy loaded params batch object
+    # @note Until first params usage, it won't parse data at all
+    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
+    #   content (from Kafka) or messages inside a hash (from backend, etc)
+    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
+    def params_batch=(messages)
+      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    end
+
+    # Executes the default consumer flow.
+    def call
+      process
+    end
+
+    private
+
+    # We make it private as it should be accessible only from the inside of a consumer
+    attr_reader :params_batch
+
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
+    #    commit manually offset or pause / stop consumer based on the business logic
+    def client
+      Persistence::Client.read
+    end
+
+    # Method that will perform business logic and on data received from Kafka (it will consume
+    #   the data)
+    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    #   someone forgets about it or makes on with typo
+    def consume
+      raise NotImplementedError, 'Implement this in a subclass'
+    end
+  end
+end

data/lib/karafka/base_responder.rb

@@ -62,6 +62,11 @@ module Karafka
     # Definitions of all topics that we want to be able to use in this responder should go here
     class_attribute :topics
 
+    # Schema that we can use to control and/or require some additional details upon options
+    # that are being passed to the producer. This can be in particular useful if we want to make
+    # sure that for example partition_key is always present.
+    class_attribute :options_schema
+
     attr_reader :messages_buffer
 
     class << self
@@ -92,7 +97,7 @@ module Karafka
     # @param parser_class [Class] parser class that we can use to generate appropriate string
     #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::Parsers::Json)
+    def initialize(parser_class = Karafka::App.config.parser)
       @parser_class = parser_class
       @messages_buffer = {}
     end
@@ -108,7 +113,8 @@ module Karafka
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
-      validate!
+      validate_usage!
+      validate_options!
       deliver!
     end
 
@@ -116,7 +122,7 @@ module Karafka
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
-    def validate!
+    def validate_usage!
       registered_topics = self.class.topics.map do |name, topic|
         topic.to_h.merge!(
           usage_count: messages_buffer[name]&.count || 0
@@ -138,21 +144,26 @@ module Karafka
       raise Karafka::Errors::InvalidResponderUsage, result.errors
     end
 
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_schema
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_schema.call(message_data.last)
+          next if result.success?
+          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+        end
+      end
+    end
+
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics
     def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
-        data_elements.each do |(data, options)|
-          ::WaterDrop::Message.new(
-            mapped_topic,
-            data,
-            options
-          ).send!
+      messages_buffer.each_value do |data_elements|
+        data_elements.each do |data, options|
+          producer(options).call(data, options)
         end
       end
     end
@@ -171,10 +182,23 @@ module Karafka
     # @param options [Hash] options for waterdrop (e.g. partition_key)
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
-      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
+
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        @parser_class.generate(data),
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
+      ]
+    end
 
-      messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+    # @param options [Hash] options for waterdrop
+    # @return [Class] WaterDrop producer (sync or async based on the settings)
+    def producer(options)
+      options[:async] ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
     end
   end
 end

data/lib/karafka/callbacks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional callbacks that are used to trigger some things in given places during the
+  # system lifecycle
+  # @note Those callbacks aren't the same as consumer callbacks as they are not related to the
+  #   lifecycle of particular messages fetches but rather to the internal flow process.
+  #   They cannot be defined on a consumer callback level because for some of those,
+  #   there aren't consumers in the memory yet and/or they aren't per consumer thread
+  module Callbacks
+    # Types of system callbacks that we have that are not related to consumers
+    TYPES = %i[
+      after_init
+      before_fetch_loop
+    ].freeze
+
+    class << self
+      TYPES.each do |callback_type|
+        # Executes given callbacks set at a given moment with provided arguments
+        define_method callback_type do |*args|
+          Karafka::App
+            .config
+            .callbacks
+            .send(callback_type)
+            .each { |callback| callback.call(*args) }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/callbacks/config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # Additional configuration required to store procs that we will execute upon callback trigger
+    module Config
+      # Builds up internal callback accumulators
+      # @param klass [Class] Class that we extend with callback config
+      def self.extended(klass)
+        # option internal [Hash] - optional - internal karafka configuration settings that should
+        #   never be changed by users directly
+        klass.setting :callbacks do
+          Callbacks::TYPES.each do |callback_type|
+            # option [Array<Proc>] an array of blocks that will be executed at a given moment
+            #   depending on the callback type
+            setting callback_type, []
+          end
+        end
+      end
+    end
+  end
+end