karafka 1.2.13 → 1.3.0.rc1

data/README.md

@@ -2,6 +2,14 @@
 
 [![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
 
+## New release in progress!
+
+**Note**: Documentation presented here refers to not yet released Karafka `1.3.0`. We're in the process of crafting the gems.
+
+If you are looking for the documentation for Karafka `1.2.*`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
+
+## About Karafka
+
 Framework used to simplify Apache Kafka based Ruby applications development.
 
 Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming and processing this data, without having to focus on things that are not your business domain.
@@ -27,15 +35,6 @@ Karafka based applications can be easily deployed to any type of infrastructure,
 * Docker
 * Terraform
 
-## Kafka 0.10 or prior
-
-If you're using Kafka 0.10, please lock `ruby-kafka` gem in your Gemfile to version `0.6.8`:
-
-```ruby
-gem 'karafka'
-gem 'ruby-kafka', '~> 0.6.8'
-```
-
 ## Support
 
 Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
@@ -52,7 +51,7 @@ If you're completely new to the subject, you can start with our "Kafka on Rails"
 If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
 
 ```bash
-git clone https://github.com/karafka/karafka-example-app ./example_app
+git clone https://github.com/karafka/example-app ./example_app
 ```
 
 then, just bundle install all the dependencies:
@@ -62,7 +61,7 @@ cd ./example_app
 bundle install
 ```
 
-and follow the instructions from the [example app Wiki](https://github.com/karafka/karafka-example-app/blob/master/README.md).
+and follow the instructions from the [example app Wiki](https://github.com/karafka/example-app/blob/master/README.md).
 
 **Note**: you need to ensure, that you have Kafka up and running and you need to configure Kafka seed_brokers in the ```karafka.rb``` file.
 

data/bin/karafka

@@ -10,7 +10,7 @@ else
   # However when it is unavailable, we still want to be able to run help command
   # and install command as they don't require configured app itself to run
   raise(
-    Karafka::Errors::MissingBootFile,
+    Karafka::Errors::MissingBootFileError,
     Karafka.boot_file
   ) unless %w[-h install].include?(ARGV[0])
 end

data/certs/mensfeld.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+-----END CERTIFICATE-----

data/config/errors.yml

@@ -1,6 +1,39 @@
 en:
-  errors:
-    broker_schema?: >
-      has an invalid format.
-      Expected schema, host and port number.
-      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+  dry_validation:
+    errors:
+      invalid_broker_schema: >
+        has an invalid format
+        Expected schema, host and port number
+        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+      invalid_certificate: >
+        is not a valid certificate
+      invalid_certificate_from_path: >
+        is not a valid certificate
+      invalid_private_key: >
+        is not a valid private key
+      max_timeout_size_for_exponential: >
+        pause_timeout cannot be more than pause_max_timeout
+      max_wait_time_limit:
+        max_wait_time cannot be more than socket_timeout
+      topics_names_not_unique: >
+        all topic names within a single consumer group must be unique
+      ssl_client_cert_with_ssl_client_cert_key: >
+        Both ssl_client_cert and ssl_client_cert_key need to be provided
+      ssl_client_cert_key_with_ssl_client_cert: >
+        Both ssl_client_cert_key and ssl_client_cert need to be provided
+      ssl_client_cert_chain_with_ssl_client_cert: >
+        Both ssl_client_cert_chain and ssl_client_cert need to be provided
+      ssl_client_cert_chain_with_ssl_client_cert_key: >
+        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided
+      ssl_client_cert_key_password_with_ssl_client_cert_key: >
+        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided
+      does_not_respond_to_token: >
+        needs to respond to a #token method
+      required_usage_count: >
+        Given topic must be used at least once
+      pid_already_exists: >
+        Pidfile already exists
+      consumer_groups_inclusion: >
+        Unknown consumer group
+      does_not_exist:
+        Given file does not exist or cannot be read

data/karafka.gemspec

@@ -5,6 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
 
+# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
   spec.version     = ::Karafka::VERSION
@@ -16,27 +17,28 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'activesupport', '>= 4.0'
   spec.add_dependency 'dry-configurable', '~> 0.8'
   spec.add_dependency 'dry-inflector', '~> 0.1'
   spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 0.11'
+  spec.add_dependency 'dry-validation', '~> 1.2'
   spec.add_dependency 'envlogic', '~> 1.0'
+  spec.add_dependency 'irb', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'require_all', '>= 1.4'
-  spec.add_dependency 'ruby-kafka', '>= 0.6'
+  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
   spec.add_dependency 'thor', '~> 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.2.4'
+  spec.add_dependency 'waterdrop', '~> 1.3.0.rc1'
+  spec.add_dependency 'zeitwerk', '~> 2.1'
 
-  spec.post_install_message = <<~MSG
-    \e[93mWarning:\e[0m If you're using Kafka 0.10, please lock ruby-kafka in your Gemfile to version '0.6.8':
-    gem 'ruby-kafka', '~> 0.6.8'
-  MSG
+  spec.required_ruby_version = '>= 2.4.0'
 
-  spec.required_ruby_version = '>= 2.3.0'
+  if $PROGRAM_NAME.end_with?('gem')
+    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
 
+  spec.cert_chain    = %w[certs/mensfeld.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
 end
+# rubocop:enable Metrics/BlockLength

data/lib/karafka.rb

@@ -9,13 +9,13 @@
   forwardable
   fileutils
   multi_json
-  require_all
   dry-configurable
   dry-validation
+  dry/events/publisher
   dry/inflector
   dry/monitor/notifications
-  active_support/callbacks
-  karafka/loader
+  dry/core/constants
+  zeitwerk
 ].each(&method(:require))
 
 # Karafka library
@@ -63,17 +63,9 @@ module Karafka
   end
 end
 
-%w[
-  callbacks
-  callbacks/*
-  setup/dsl
-  setup/config
-  status
-  schemas/config
-  schemas/consumer_group_topic
-  schemas/consumer_group
-].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
+Zeitwerk::Loader
+  .for_gem
+  .tap(&:setup)
+  .tap(&:eager_load)
 
-Karafka::Loader.load!(Karafka.core_root)
 Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
-Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/app.rb

@@ -4,28 +4,36 @@ module Karafka
   # App class
   class App
     extend Setup::Dsl
-    extend Callbacks::Dsl
 
     class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
-      #   doesn't match with ConfigurationSchema
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
       def boot!
+        initialize!
         Setup::Config.validate!
         Setup::Config.setup_components
-        Callbacks.after_init(Karafka::App.config)
+        initialized!
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
       def consumer_groups
-        Routing::Builder.instance
+        config.internal.routing_builder
+      end
+
+      # Triggers reload of all cached Karafka app components, so we can use in-process
+      # in-development hot code reloading without Karafka process restart
+      def reload
+        Karafka::Persistence::Consumers.clear
+        Karafka::Persistence::Topics.clear
+        config.internal.routing_builder.reload
       end
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.send(delegated)
+          App.config.internal.status.send(delegated)
         end
       end
 

data/lib/karafka/attributes_map.rb

@@ -22,7 +22,7 @@ module Karafka
           ],
           subscribe: %i[start_from_beginning max_bytes_per_partition],
           consumption: %i[min_bytes max_bytes max_wait_time],
-          pause: %i[pause_timeout],
+          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -35,10 +35,9 @@ module Karafka
         (api_adapter[:subscribe] + %i[
           backend
           name
-          parser
+          deserializer
           responder
           batch_consuming
-          persistent
         ]).uniq
       end
 
@@ -53,7 +52,7 @@ module Karafka
         ignored_settings = api_adapter[:subscribe]
         defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
-        # This is a drity and bad hack of dry-configurable to get keys before setting values
+        # This is a dirty and bad hack of dry-configurable to get keys before setting values
         dynamically_proxied = Karafka::Setup::Config
                               ._settings
                               .settings

data/lib/karafka/base_consumer.rb

@@ -4,41 +4,33 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend ActiveSupport::DescendantsTracker
     extend Forwardable
 
     # Allows us to mark messages as consumed for non-automatic mode without having
     # to use consumer client directly. We do this that way, because most of the people should not
     # mess with the client instance directly (just in case)
-    def_delegator :client, :mark_as_consumed
-
-    private :mark_as_consumed
-
-    class << self
-      attr_reader :topic
-
-      # Assigns a topic to a consumer and builds up proper consumer functionalities
-      #   so that it can cooperate with the topic settings
-      # @param topic [Karafka::Routing::Topic]
-      # @return [Karafka::Routing::Topic] assigned topic
-      def topic=(topic)
-        @topic = topic
-        Consumers::Includer.call(self)
-      end
+    %i[
+      mark_as_consumed
+      mark_as_consumed!
+      trigger_heartbeat
+      trigger_heartbeat!
+    ].each do |delegated_method_name|
+      def_delegator :client, delegated_method_name
+
+      private delegated_method_name
     end
 
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    def topic
-      self.class.topic
-    end
-
-    # Creates lazy loaded params batch object
-    # @note Until first params usage, it won't parse data at all
-    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
-    #   content (from Kafka) or messages inside a hash (from backend, etc)
-    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
-    def params_batch=(messages)
-      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    attr_reader :topic
+    # @return [Karafka::Params:ParamsBatch] current params batch
+    attr_accessor :params_batch
+
+    # Assigns a topic to a consumer and builds up proper consumer functionalities
+    #   so that it can cooperate with the topic settings
+    # @param topic [Karafka::Routing::Topic]
+    def initialize(topic)
+      @topic = topic
+      Consumers::Includer.call(self)
     end
 
     # Executes the default consumer flow.
@@ -48,9 +40,6 @@ module Karafka
 
     private
 
-    # We make it private as it should be accessible only from the inside of a consumer
-    attr_reader :params_batch
-
     # @return [Karafka::Connection::Client] messages consuming client that can be used to
     #    commit manually offset or pause / stop consumer based on the business logic
     def client

data/lib/karafka/base_responder.rb

@@ -39,7 +39,7 @@ module Karafka
   #
   # @example Multiple times used topic
   #   class Responder < BaseResponder
-  #     topic :required_topic, multiple_usage: true
+  #     topic :required_topic
   #
   #     def respond(data)
   #       data.each do |subset|
@@ -48,6 +48,17 @@ module Karafka
   #     end
   #   end
   #
+  # @example Specify serializer for a topic
+  #   class Responder < BaseResponder
+  #     topic :xml_topic, serializer: MyXMLSerializer
+  #
+  #     def respond(data)
+  #       data.each do |subset|
+  #         respond_to :xml_topic, subset
+  #       end
+  #     end
+  #   end
+  #
   # @example Accept multiple arguments to a respond method
   #   class Responder < BaseResponder
   #     topic :users_actions
@@ -59,31 +70,35 @@ module Karafka
   #     end
   #   end
   class BaseResponder
-    # Definitions of all topics that we want to be able to use in this responder should go here
-    class_attribute :topics
-
-    # Schema that we can use to control and/or require some additional details upon options
-    # that are being passed to the producer. This can be in particular useful if we want to make
-    # sure that for example partition_key is always present.
-    class_attribute :options_schema
+    # Responder usage contract
+    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
 
-    attr_reader :messages_buffer
+    private_constant :CONTRACT
 
     class << self
+      # Definitions of all topics that we want to be able to use in this responder should go here
+      attr_accessor :topics
+      # Contract that we can use to control and/or require some additional details upon options
+      # that are being passed to the producer. This can be in particular useful if we want to make
+      # sure that for example partition_key is always present.
+      attr_accessor :options_contract
+
       # Registers a topic as on to which we will be able to respond
       # @param topic_name [Symbol, String] name of topic to which we want to respond
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
+        options[:serializer] ||= Karafka::App.config.serializer
+        options[:registered] = true
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
+        topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
 
       # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # Instead of building it with new.call it allows (in case of using JSON serializer)
       # to just run it directly from the class level
       # @param data Anything that we want to respond with
-      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      # @example Send user data with a responder
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
         # Just in case there were no topics defined for a responder, we initialize with
@@ -93,12 +108,11 @@ module Karafka
       end
     end
 
+    attr_reader :messages_buffer
+
     # Creates a responder object
-    # @param parser_class [Class] parser class that we can use to generate appropriate string
-    #   or nothing if we want to default to Karafka::Parsers::Json
     # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize(parser_class = Karafka::App.config.parser)
-      @parser_class = parser_class
+    def initialize
       @messages_buffer = {}
     end
 
@@ -107,7 +121,7 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
-    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    # @example Send user data with a responder
     #   UsersCreatedResponder.new.call(@created_user)
     # @example Send user data with a responder using non default Parser
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
@@ -134,25 +148,26 @@ module Karafka
         topic.to_h.merge!(usage_count: usage.count)
       end
 
-      result = Karafka::Schemas::ResponderUsage.call(
+      result = CONTRACT.call(
         registered_topics: registered_topics,
         used_topics: used_topics
       )
 
       return if result.success?
 
-      raise Karafka::Errors::InvalidResponderUsage, result.errors
+      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
     end
 
     # Checks if we met all the options requirements before sending them to the producer.
     def validate_options!
-      return true unless self.class.options_schema
+      return true unless self.class.options_contract
 
       messages_buffer.each_value do |messages_set|
         messages_set.each do |message_data|
-          result = self.class.options_schema.call(message_data.last)
+          result = self.class.options_contract.call(message_data.last)
           next if result.success?
-          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+
+          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
         end
       end
     end
@@ -174,6 +189,7 @@ module Karafka
 
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
+    # @param _data [Object] anything that we want to use to send to Kafka
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
     def respond(*_data)
       raise NotImplementedError, 'Implement this in a subclass'
@@ -183,7 +199,7 @@ module Karafka
     # as many times as we need. Especially when we have 1:n flow
     # @param topic [Symbol, String] topic to which we want to respond
     # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key)
+    # @param options [Hash] options for waterdrop (e.g. partition_key).
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
       # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
@@ -192,7 +208,7 @@ module Karafka
 
       messages_buffer[topic] ||= []
       messages_buffer[topic] << [
-        @parser_class.generate(data),
+        self.class.topics[topic].serializer.call(data),
         options.merge(topic: topic)
       ]
     end
@@ -200,9 +216,11 @@ module Karafka
     # @param options [Hash] options for waterdrop
     # @return [Class] WaterDrop producer (sync or async based on the settings)
     def producer(options)
-      self.class.topics[
-        options[:topic]
-      ].async? ? WaterDrop::AsyncProducer : WaterDrop::SyncProducer
+      if self.class.topics[options[:topic]].async?
+        WaterDrop::AsyncProducer
+      else
+        WaterDrop::SyncProducer
+      end
     end
   end
 end