karafka 2.0.0.alpha1 → 2.0.0.alpha4

data/lib/karafka/contracts/config.rb

@@ -8,9 +8,7 @@ module Karafka
     #   `Karafka::Setup::Config` model, but we don't validate them here as they are
     #   validated per each route (topic + consumer_group) because they can be overwritten,
     #   so we validate all of that once all the routes are defined and ready.
-    class Config < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class Config < Base
       params do
         # License validity happens in the licenser. Here we do only the simple consistency checks
         required(:license).schema do
@@ -27,6 +25,26 @@ module Karafka
         required(:pause_max_timeout) { int? & gt?(0) }
         required(:pause_with_exponential_backoff).filled(:bool?)
         required(:shutdown_timeout) { int? & gt?(0) }
+        required(:kafka).filled(:hash)
+
+        # We validate internals just to be sure, that they are present and working
+        required(:internal).schema do
+          required(:routing_builder)
+          required(:status)
+          required(:process)
+          required(:subscription_groups_builder)
+        end
+      end
+
+      # rdkafka requires all the keys to be strings, so we ensure that
+      rule(:kafka) do
+        next unless value.is_a?(Hash)
+
+        value.each_key do |key|
+          next if key.is_a?(Symbol)
+
+          key(:"kafka.#{key}").failure(:kafka_key_must_be_a_symbol)
+        end
       end
 
       rule(:pause_timeout, :pause_max_timeout) do

data/lib/karafka/contracts/consumer_group.rb

@@ -3,9 +3,7 @@
 module Karafka
   module Contracts
     # Contract for single full route (consumer group + topics) validation.
-    class ConsumerGroup < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ConsumerGroup < Base
       # Internal contract for sub-validating topics schema
       TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
 

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -3,15 +3,14 @@
 module Karafka
   module Contracts
     # Consumer group topic validation rules.
-    class ConsumerGroupTopic < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ConsumerGroupTopic < Base
       params do
         required(:consumer).filled
         required(:deserializer).filled
         required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
         required(:kafka).filled
         required(:max_messages) { int? & gteq?(1) }
+        required(:initial_offset).filled(included_in?: %w[earliest latest])
         required(:max_wait_time).filled { int? & gteq?(10) }
         required(:manual_offset_management).filled(:bool?)
         required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)

data/lib/karafka/contracts/server_cli_options.rb

@@ -3,9 +3,7 @@
 module Karafka
   module Contracts
     # Contract for validating correctness of the server cli command options.
-    class ServerCliOptions < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ServerCliOptions < Base
       params do
         optional(:consumer_groups).value(:array, :filled?)
       end

data/lib/karafka/errors.rb

@@ -43,5 +43,9 @@ module Karafka
 
     # Raised when the license token is not valid
     InvalidLicenseTokenError = Class.new(BaseError)
+
+    # Used to instrument this error into the error notifications
+    # We do not raise it so we won't crash deployed systems
+    ExpiredLicenseTokenError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/monitor.rb

@@ -17,6 +17,7 @@ module Karafka
       #   complete list of all the events. Please use the #available_events on fully loaded
       #   Karafka system to determine all of the events you can use.
       BASE_EVENTS = %w[
+        app.initialized
         app.running
         app.stopping
         app.stopped

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -88,6 +88,9 @@ module Karafka
         when 'connection.listener.fetch_loop.error'
           error "Listener fetch loop error: #{error}"
           error details
+        when 'licenser.expired'
+          error error
+          error details
         when 'runner.call.error'
           fatal "Runner crashed due to an error: #{error}"
           fatal details

data/lib/karafka/licenser.rb

@@ -26,7 +26,7 @@ module Karafka
         data = nil
       end
 
-      details = data ? JSON.parse(data) : raise_invalid_license_token
+      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
 
       license_config.entity = details.fetch('entity')
       license_config.expires_on = Date.parse(details.fetch('expires_on'))
@@ -39,7 +39,11 @@ module Karafka
     private
 
     # Raises an error with info, that used token is invalid
-    def raise_invalid_license_token
+    # @param license_config [Dry::Configurable::Config]
+    def raise_invalid_license_token(license_config)
+      # We set it to false so `Karafka.pro?` method behaves as expected
+      license_config.token = false
+
       raise(
         Errors::InvalidLicenseTokenError,
         <<~MSG.tr("\n", ' ')
@@ -50,15 +54,22 @@ module Karafka
     end
 
     # We do not raise an error here as we don't want to cause any problems to someone that runs
-    # Karafka on production. Error is enough.
+    # Karafka on production. Error message is enough.
+    #
     # @param expires_on [Date] when the license expires
     def notify_if_license_expired(expires_on)
-      Karafka.logger.error(
-        <<~MSG.tr("\n", ' ')
-          Your license expired on #{expires_on}.
-          Karafka no longer uses any Pro capabilities.
-          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-        MSG
+      message = <<~MSG.tr("\n", ' ')
+        Your license expired on #{expires_on}.
+        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+      MSG
+
+      Karafka.logger.error(message)
+
+      Karafka.monitor.instrument(
+        'error.occurred',
+        caller: self,
+        error: Errors::ExpiredLicenseTokenError.new(message),
+        type: 'licenser.expired'
       )
     end
   end

data/lib/karafka/messages/batch_metadata.rb

@@ -14,6 +14,8 @@ module Karafka
       :partition,
       :topic,
       :scheduled_at,
+      :consumption_lag,
+      :processing_lag,
       keyword_init: true
     )
   end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -12,7 +12,12 @@ module Karafka
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
+          # @note Regarding the time lags: we can use the current time here, as batch metadata is
+          #   created in the worker. So whenever this is being built, it means that the processing
+          #   of this batch has already started.
           def call(kafka_batch, topic, scheduled_at)
+            now = Time.now
+
             Karafka::Messages::BatchMetadata.new(
               size: kafka_batch.count,
               first_offset: kafka_batch.first.offset,
@@ -20,9 +25,26 @@ module Karafka
               deserializer: topic.deserializer,
               partition: kafka_batch[0].partition,
               topic: topic.name,
-              scheduled_at: scheduled_at
+              scheduled_at: scheduled_at,
+              # This lag describes how long did it take for a message to be consumed from the
+              # moment it was created
+              consumption_lag: time_distance_in_ms(now, kafka_batch.last.timestamp),
+              # This lag describes how long did a batch have to wait before it was picked up by
+              # one of the workers
+              processing_lag: time_distance_in_ms(now, scheduled_at)
             ).freeze
           end
+
+          private
+
+          # Computes time distance in between two times in ms
+          #
+          # @param time1 [Time]
+          # @param time2 [Time]
+          # @return [Integer] distance in between two times in ms
+          def time_distance_in_ms(time1, time2)
+            ((time1 - time2) * 1_000).round
+          end
         end
       end
     end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Karafka Pro ActiveJob components
+    module ActiveJob
+      # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+      # and that allows to inject additional options into the producer, effectively allowing for a
+      # much better and more granular control over the dispatch and consumption process.
+      class Dispatcher < ::Karafka::ActiveJob::Dispatcher
+        # Defaults for dispatching
+        # The can be updated by using `#karafka_options` on the job
+        DEFAULTS = {
+          dispatch_method: :produce_async,
+          # We don't create a dummy proc based partitioner as we would have to evaluate it with
+          # each job.
+          partitioner: nil
+        }.freeze
+
+        private_constant :DEFAULTS
+
+        # @param job [ActiveJob::Base] job
+        def call(job)
+          ::Karafka.producer.public_send(
+            fetch_option(job, :dispatch_method, DEFAULTS),
+            dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(job.serialize)
+            )
+          )
+        end
+
+        private
+
+        # @param job [ActiveJob::Base] job instance
+        # @return [Hash] hash with dispatch details to which we merge topic and payload
+        def dispatch_details(job)
+          partitioner = fetch_option(job, :partitioner, DEFAULTS)
+
+          return {} unless partitioner
+
+          {
+            partition_key: partitioner.call(job)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # Contract for validating the options that can be altered with `#karafka_options` per job
+      # class that works with Pro features.
+      class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract
+        # Dry types
+        Types = include Dry.Types()
+
+        params do
+          optional(:partitioner).value(Types.Interface(:call))
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+module Karafka
+  module Pro
+    # Loader requires and loads all the pro components only when they are needed
+    class Loader
+      class << self
+        # Loads all the pro components and configures them wherever it is expected
+        # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
+        #   components
+        def setup(config)
+          require_relative 'active_job/dispatcher'
+          require_relative 'active_job/job_options_contract'
+
+          config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
+          config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+module Karafka
+  # Namespace for pro components, licensed under the commercial license agreement.
+  module Pro
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -41,7 +41,7 @@ module Karafka
       # We signal critical exceptions, notify and do not allow worker to fail
       # rubocop:disable Lint/RescueException
       rescue Exception => e
-      # rubocop:enable Lint/RescueException
+        # rubocop:enable Lint/RescueException
         Karafka.monitor.instrument(
           'error.occurred',
           caller: self,

data/lib/karafka/railtie.rb

@@ -19,6 +19,7 @@ end
 if rails
   # Load Karafka
   require 'karafka'
+
   # Load ActiveJob adapter
   require 'active_job/karafka'
 
@@ -30,34 +31,69 @@ if rails
     class Railtie < Rails::Railtie
       railtie_name :karafka
 
-      initializer 'karafka.configure_rails_initialization' do |app|
-        # Consumers should autoload by default in the Rails app so they are visible
-        app.config.autoload_paths += %w[app/consumers]
+      initializer 'karafka.active_job_integration' do
+        ActiveSupport.on_load(:active_job) do
+          # Extend ActiveJob with some Karafka specific ActiveJob magic
+          extend ::Karafka::ActiveJob::JobExtensions
+        end
+      end
 
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
         # Make Karafka use Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
-        # This lines will make Karafka print to stdout like puma or unicorn
-        if Rails.env.development?
-          Rails.logger.extend(
-            ActiveSupport::Logger.broadcast(
-              ActiveSupport::Logger.new($stdout)
-            )
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+
+        Rails.logger.extend(
+          ActiveSupport::Logger.broadcast(
+            ActiveSupport::Logger.new($stdout)
           )
+        )
+      end
 
-          # We can have many listeners, but it does not matter in which we will reload the code as
-          # long as all the consumers will be re-created as Rails reload is thread-safe
-          ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-            # Reload code each time there is a change in the code
-            next unless Rails.application.reloaders.any?(&:updated?)
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
 
-            Rails.application.reloader.reload!
-          end
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
         end
+      end
 
-        app.reloader.to_prepare do
-          # Load Karafka bot file, so it can be used in Rails server context
-          require Rails.root.join(Karafka.boot_file.to_s).to_s
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
         end
       end
     end

data/lib/karafka/routing/builder.rb

@@ -10,11 +10,6 @@ module Karafka
     #     end
     #   end
     class Builder < Concurrent::Array
-      # Consumer group consistency checking contract
-      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
-
-      private_constant :CONTRACT
-
       def initialize
         @draws = Concurrent::Array.new
         super
@@ -38,12 +33,7 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
-          hashed_group = consumer_group.to_h
-          validation_result = CONTRACT.call(hashed_group)
-
-          next if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
         end
       end
 

data/lib/karafka/routing/subscription_group.rb

@@ -20,7 +20,7 @@ module Karafka
 
       # @return [String] consumer group id
       def consumer_group_id
-        kafka['group.id']
+        kafka[:'group.id']
       end
 
       # @return [Integer] max messages fetched in a single go
@@ -39,12 +39,12 @@ module Karafka
       def kafka
         kafka = @topics.first.kafka.dup
 
-        kafka['client.id'] ||= Karafka::App.config.client_id
-        kafka['group.id'] ||= @topics.first.consumer_group.id
-        kafka['auto.offset.reset'] ||= 'earliest'
+        kafka[:'client.id'] ||= Karafka::App.config.client_id
+        kafka[:'group.id'] ||= @topics.first.consumer_group.id
+        kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka['enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = 'false'
         kafka.freeze
         kafka
       end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -18,6 +18,7 @@ module Karafka
         kafka
         max_messages
         max_wait_time
+        initial_offset
       ].freeze
 
       private_constant :DISTRIBUTION_KEYS

data/lib/karafka/routing/topic.rb

@@ -16,6 +16,7 @@ module Karafka
         manual_offset_management
         max_messages
         max_wait_time
+        initial_offset
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES

data/lib/karafka/setup/config.rb

@@ -14,15 +14,12 @@ module Karafka
     class Config
       extend Dry::Configurable
 
-      # Contract for checking the config provided by the user
-      CONTRACT = Karafka::Contracts::Config.new.freeze
-
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id' => 'karafka'
+        'client.id': 'karafka'
       }.freeze
 
-      private_constant :CONTRACT, :KAFKA_DEFAULTS
+      private_constant :KAFKA_DEFAULTS
 
       # Available settings
 
@@ -57,6 +54,9 @@ module Karafka
       setting :consumer_persistence, default: true
       # Default deserializer for converting incoming data into ruby objects
       setting :deserializer, default: Karafka::Serialization::Json::Deserializer.new
+      # option [String] should we start with the earliest possible offset or latest
+      # This will set the `auto.offset.reset` value unless present in the kafka scope
+      setting :initial_offset, default: 'earliest'
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
@@ -84,8 +84,6 @@ module Karafka
       setting :kafka, default: {}
 
       # Namespace for internal settings that should not be modified
-      # It's a temporary step to "declassify" several things internally before we move to a
-      # non global state
       setting :internal do
         # option routing_builder [Karafka::Routing::Builder] builder instance
         setting :routing_builder, default: Routing::Builder.new
@@ -98,6 +96,17 @@ module Karafka
         # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
         #   group builder
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+        # Karafka components for ActiveJob
+        setting :active_job do
+          # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
+          setting :dispatcher, default: ActiveJob::Dispatcher.new
+          # option job_options_contract [Karafka::Contracts::JobOptionsContract] contract for
+          #   ensuring, that extra job options defined are valid
+          setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
+          # option consumer [Class] consumer class that should be used to consume ActiveJob data
+          setting :consumer, default: ActiveJob::Consumer
+        end
       end
 
       class << self
@@ -106,12 +115,14 @@ module Karafka
         def setup(&block)
           configure(&block)
           merge_kafka_defaults!(config)
-          validate!
+          Contracts::Config.new.validate!(config.to_h)
 
           # Check the license presence (if needed) and
           Licenser.new.verify(config.license)
 
           configure_components
+
+          Karafka::App.initialized!
         end
 
         private
@@ -128,18 +139,6 @@ module Karafka
           end
         end
 
-        # Validate config based on the config contract
-        # @return [Boolean] true if configuration is valid
-        # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-        #   doesn't match with the config contract
-        def validate!
-          validation_result = CONTRACT.call(config.to_h)
-
-          return true if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
-        end
-
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components
@@ -149,6 +148,12 @@ module Karafka
             producer_config.kafka = config.kafka.dup
             producer_config.logger = config.logger
           end
+
+          return unless Karafka.pro?
+
+          # Runs the pro loader that includes all the pro components
+          require 'karafka/pro/loader'
+          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/status.rb

@@ -6,6 +6,7 @@ module Karafka
     # Available states and their transitions.
     STATES = {
       initializing: :initialize!,
+      initialized: :initialized!,
       running: :run!,
       stopping: :stop!,
       stopped: :stopped!

data/lib/karafka/templates/karafka.rb.erb

@@ -27,7 +27,7 @@ APP_LOADER.eager_load
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka = { 'bootstrap.servers' => '127.0.0.1:9092' }
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
     config.client_id = 'example_app'
 <% if rails? -%>
     # Recreate consumers with each batch. This will allow Rails code reload to work in the

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha1'
+  VERSION = '2.0.0.alpha4'
 end