karafka 2.0.0.alpha1 → 2.0.0.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a441a124346099aa31fb36a2241c2abfbc242ab6f3f9ad197b82d8786e78232e
-  data.tar.gz: 81a706d58e469ecc48759dfe28c30e3779baab848122b72c194c30e06a7cdd7d
+  metadata.gz: 112496d14f74a8095df2b0b557e294140f422bbc8d6ea93f0709ac3b4ec017ec
+  data.tar.gz: a049427bd67d7887c0d5a352c724ecc56b156842859e9688068d36e25aeeb05d
 SHA512:
-  metadata.gz: bf9ebff340150913cf273bc3f09c9f3dd3c02e6ffe427fd00ead842995e5f146d554609cd1ff4d1039ee96ef349b8e7712e2c557abc322a9e0300886bc765b6a
-  data.tar.gz: 054cf8240a33a133c2c9dce5f1c3c2ac26877006c8a823c1e58c903fccd8cac7c31b5d0c0d2ec22257d8959d40bb69d49bf5676f378c33004807a78413278b3e
+  metadata.gz: d549e3c4bd31a2d011d453495ee6c3f3825ece067285834ef1bc5872cbc257f9695ff831eed7370c0668b33f4c297b1c3512fb068fecafdbd1a5d109d305e243
+  data.tar.gz: 2f7a48f6490e7ccef765c1697bdb54eb8e596246c853f8e30a85a134e50394c78c983b2ad4020d140da0a532aa1633cd425ba3af12a2d1e2f59fc9264288c3fb

data/.github/workflows/ci.yml

@@ -122,5 +122,6 @@ jobs:
 
       - name: Run integration tests
         env:
+          KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bundle exec bin/integrations

data/CHANGELOG.md

@@ -1,6 +1,9 @@
 # Karafka framework changelog
 
-## 2.0.0-alpha1 (Unreleased)
+## 2.0.0-alpha2 (2022-02-19)
+- Require `kafka` keys to be symbols
+
+## 2.0.0-alpha1 (2022-01-30)
 - Change license to `LGPL-3.0`
 - Introduce a Pro subscription
 - Switch from `ruby-kafka` to `librdkafka` as an underlying driver

data/CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 ## Introduction
 
-First, thank you for considering contributing to karafka! It's people like you that make the open source community such a great community! 😊
+First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community! 😊
 
 We welcome any type of contribution, not only code. You can help with:
 - **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
@@ -18,15 +18,17 @@ Working on your first Pull Request? You can learn how from this *free* series, [
 
 Any code change should be submitted as a pull request. The description should explain what the code does and give steps to execute it. The pull request should also contain tests.
 
-## Code review process
+### Code review process
 
 Each pull request must pass all the rspec specs and meet our quality requirements.
 
 To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 
-Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/commit_builds) of Karafka organization.
+### Contributing to Pro components
 
-[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/commit_builds)
+All of Karafka components are open-source. However, the `Pro` components are licenses under `LICENSE-COMM`.
+
+By sending a pull request to the pro components, you are agreeing to transfer the copyright of your code to Maciej Mensfeld.
 
 ## Questions
 
@@ -35,7 +37,5 @@ You can also reach us at hello@karafka.opencollective.com.
 
 ## Credits
 
-### Contributors
-
 Thank you to all the people who have already contributed to karafka!
 <a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>

data/Gemfile.lock

@@ -1,22 +1,22 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.pre.alpha1)
+    karafka (2.0.0.alpha2)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       thor (>= 0.20)
-      waterdrop (>= 2.1.0, < 3.0.0)
+      waterdrop (>= 2.2.0, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.1)
-      activesupport (= 7.0.1)
+    activejob (7.0.2.2)
+      activesupport (= 7.0.2.2)
       globalid (>= 0.3.6)
-    activesupport (7.0.1)
+    activesupport (7.0.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -45,7 +45,7 @@ GEM
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.8.0)
+    dry-schema (1.9.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
@@ -58,18 +58,18 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.7.0)
+    dry-validation (1.8.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.8, >= 1.8.0)
+      dry-schema (~> 1.9, >= 1.9.1)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
     globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.9.1)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.7.1)
     minitest (5.15.0)
@@ -78,29 +78,29 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.2)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.2)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
+    simplecov_json_formatter (0.1.4)
     thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.1.0)
+    waterdrop (2.2.0)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
@@ -121,4 +121,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.5
+   2.3.7

data/LICENSE

@@ -9,6 +9,9 @@ Karafka has also commercial-friendly license, commercial support and commercial
 All of the commercial components are present in the lib/karafka/pro directory of this repository
 and their usage requires commercial license agreement.
 
+By sending a pull request to the pro components, you are agreeing to transfer the copyright of your
+code to Maciej Mensfeld.
+
 You can find the commercial license in LICENSE-COM.
 
 Please see https://karafka.io for purchasing options.

data/bin/integrations

@@ -13,7 +13,7 @@ IntegrationTestError = Class.new(StandardError)
 
 # How many child processes with integration specs do we want to run in parallel
 # When the value is high, there's a problem with thread allocation on Github
-CONCURRENCY = 5
+CONCURRENCY = 4
 
 # Abstraction around a single test scenario execution process
 class Scenario

data/bin/karafka

@@ -2,6 +2,10 @@
 
 require 'karafka'
 
+# We set this to indicate, that the process in which we are (whatever it does) was started using
+# our bin/karafka cli
+ENV['KARAFKA_CLI'] = 'true'
+
 # If there is a boot file, we need to require it as we expect it to contain
 # Karafka app setup, routes, etc
 if File.exist?(Karafka.boot_file)

data/config/errors.yml

@@ -5,3 +5,4 @@ en:
       topics_names_not_unique: all topic names within a single consumer group must be unique
       required_usage_count: Given topic must be used at least once
       consumer_groups_inclusion: Unknown consumer group
+      kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols

data/docker-compose.yml

@@ -16,6 +16,7 @@ services:
       KAFKA_CREATE_TOPICS:
         "integrations_0_03:3:1,\
          integrations_1_03:3:1,\
+         integrations_2_03:3:1,\
          integrations_0_10:10:1,\
          integrations_1_10:10:1,\
          benchmarks_0_01:1:1,\

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.1.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.2.0', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'

data/lib/active_job/karafka.rb

@@ -2,8 +2,6 @@
 
 require 'active_job'
 require 'active_job/queue_adapters'
-require 'active_job/consumer'
-require 'active_job/routing_extensions'
 require 'active_job/queue_adapters/karafka_adapter'
 
 module ActiveJob
@@ -14,5 +12,9 @@ module ActiveJob
 end
 
 # We extend routing builder by adding a simple wrapper for easier jobs topics defining
-::Karafka::Routing::Builder.include ActiveJob::RoutingExtensions
-::Karafka::Routing::Proxy.include ActiveJob::RoutingExtensions
+# This needs to be extended here as it is going to be used in karafka routes, hence doing that in
+# the railtie initializer would be too late
+::Karafka::Routing::Builder.include ::Karafka::ActiveJob::RoutingExtensions
+::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::RoutingExtensions
+
+# We extend ActiveJob stuff in the railtie

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -5,16 +5,13 @@ module ActiveJob
   # ActiveJob queue adapters
   module QueueAdapters
     # Karafka adapter for enqueuing jobs
+    # This is here for ease of integration with ActiveJob.
     class KarafkaAdapter
-      # Enqueues the job by sending all the payload to a dedicated topic in Kafka that will be
-      # later on consumed by a special ActiveJob consumer
+      # Enqueues the job using the configured dispatcher
       #
       # @param job [Object] job that should be enqueued
       def enqueue(job)
-        ::Karafka.producer.produce_async(
-          topic: job.queue_name,
-          payload: ActiveSupport::JSON.encode(job.serialize)
-        )
+        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
       end
 
       # Raises info, that Karafka backend does not support scheduling jobs

data/lib/karafka/active_job/consumer.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
+    # It marks the offset after each message, so we make sure, none of the jobs is executed twice
+    class Consumer < ::Karafka::BaseConsumer
+      # Executes the ActiveJob logic
+      # @note ActiveJob does not support batches, so we just run one message after another
+      def consume
+        messages.each do |message|
+          ::ActiveJob::Base.execute(
+            # We technically speaking could set this as deserializer and reference it from the
+            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+            # the ActiveJob setup here
+            ::ActiveSupport::JSON.decode(message.raw_payload)
+          )
+
+          mark_as_consumed(message)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+    class Dispatcher
+      # Defaults for dispatching
+      # The can be updated by using `#karafka_options` on the job
+      DEFAULTS = {
+        dispatch_method: :produce_async
+      }.freeze
+
+      private_constant :DEFAULTS
+
+      # @param job [ActiveJob::Base] job
+      def call(job)
+        ::Karafka.producer.public_send(
+          fetch_option(job, :dispatch_method, DEFAULTS),
+          topic: job.queue_name,
+          payload: ::ActiveSupport::JSON.encode(job.serialize)
+        )
+      end
+
+      private
+
+      # @param job [ActiveJob::Base] job
+      # @param key [Symbol] key we want to fetch
+      # @param defaults [Hash]
+      # @return [Object] options we are interested in
+      def fetch_option(job, key, defaults)
+        job
+          .class
+          .karafka_options
+          .fetch(key, defaults.fetch(key))
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_extensions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Allows for setting karafka specific options in ActiveJob jobs
+    module JobExtensions
+      class << self
+        # Defines all the needed accessors and sets defaults
+        # @param klass [ActiveJob::Base] active job base
+        def extended(klass)
+          klass.class_attribute :_karafka_options
+          klass._karafka_options = {}
+        end
+      end
+
+      # @param new_options [Hash] additional options that allow for jobs Karafka related options
+      #   customization
+      # @return [Hash] karafka options
+      def karafka_options(new_options = {})
+        return _karafka_options if new_options.empty?
+
+        # Make sure, that karafka options that someone wants to use are valid before assigning
+        # them
+        App.config.internal.active_job.job_options_contract.validate!(new_options)
+
+        new_options.each do |name, value|
+          _karafka_options[name] = value
+        end
+
+        _karafka_options
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_options_contract.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Contract for validating the options that can be altered with `#karafka_options` per job class
+    # @note We keep this in the `Karafka::ActiveJob` namespace instead of `Karafka::Contracts` as
+    #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
+    #   all in the same place
+    class JobOptionsContract < Contracts::Base
+      params do
+        optional(:dispatch_method).value(included_in?: %i[produce_async produce_sync])
+      end
+    end
+  end
+end

data/lib/karafka/active_job/routing_extensions.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  # ActiveJob related Karafka stuff
+  module ActiveJob
+    # Routing extensions for ActiveJob
+    module RoutingExtensions
+      # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
+      # the consumer class
+      # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+      def active_job_topic(name)
+        topic(name) do
+          consumer App.config.internal.active_job.consumer
+        end
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -36,6 +36,7 @@ module Karafka
         logger
         producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)

data/lib/karafka/cli/info.rb

@@ -31,9 +31,9 @@ module Karafka
       def core_info
         config = Karafka::App.config
 
-        postfix = config.license.token ? ' + Pro' : ''
+        postfix = Karafka.pro? ? ' + Pro' : ''
 
-        info = [
+        [
           "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_VERSION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
@@ -49,7 +49,7 @@ module Karafka
       def license_info
         config = Karafka::App.config
 
-        if config.license.token
+        if Karafka.pro?
           [
             'License: Commercial',
             "License entity: #{config.license.entity}",

data/lib/karafka/cli/server.rb

@@ -5,11 +5,6 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
-      # Server config settings contract
-      CONTRACT = Contracts::ServerCliOptions.new.freeze
-
-      private_constant :CONTRACT
-
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :consumer_groups, type: :array, default: nil, aliases: :g
@@ -19,7 +14,7 @@ module Karafka
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        validate!
+        Contracts::ServerCliOptions.new.validate!(cli.options)
 
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
@@ -34,7 +29,7 @@ module Karafka
       def print_marketing_info
         Karafka.logger.info Info::BANNER
 
-        if Karafka::App.config.license.token
+        if Karafka.pro?
           Karafka.logger.info(
             "\033[0;32mThank you for investing in the Karafka Pro subscription!\033[0m\n"
           )
@@ -44,15 +39,6 @@ module Karafka
           )
         end
       end
-
-      # Checks the server cli configuration
-      # options validations in terms of app setup (topics, pid existence, etc)
-      def validate!
-        result = CONTRACT.call(cli.options)
-        return if result.success?
-
-        raise Errors::InvalidConfigurationError, result.errors.to_h
-      end
     end
   end
 end

data/lib/karafka/contracts/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Base contract for all Karafka contracts
+    class Base < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      # @param data [Hash] data for validation
+      # @return [Boolean] true if all good
+      # @raise [Errors::InvalidConfigurationError] invalid configuration error
+      # @note We use contracts only in the config validation context, so no need to add support
+      #   for multiple error classes. It will be added when it will be needed.
+      def validate!(data)
+        result = call(data)
+
+        return true if result.success?
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
+      end
+    end
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -8,9 +8,7 @@ module Karafka
     #   `Karafka::Setup::Config` model, but we don't validate them here as they are
     #   validated per each route (topic + consumer_group) because they can be overwritten,
     #   so we validate all of that once all the routes are defined and ready.
-    class Config < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class Config < Base
       params do
         # License validity happens in the licenser. Here we do only the simple consistency checks
         required(:license).schema do
@@ -27,6 +25,26 @@ module Karafka
         required(:pause_max_timeout) { int? & gt?(0) }
         required(:pause_with_exponential_backoff).filled(:bool?)
         required(:shutdown_timeout) { int? & gt?(0) }
+        required(:kafka).filled(:hash)
+
+        # We validate internals just to be sure, that they are present and working
+        required(:internal).schema do
+          required(:routing_builder)
+          required(:status)
+          required(:process)
+          required(:subscription_groups_builder)
+        end
+      end
+
+      # rdkafka requires all the keys to be strings, so we ensure that
+      rule(:kafka) do
+        next unless value.is_a?(Hash)
+
+        value.each_key do |key|
+          next if key.is_a?(Symbol)
+
+          key(:"kafka.#{key}").failure(:kafka_key_must_be_a_symbol)
+        end
       end
 
       rule(:pause_timeout, :pause_max_timeout) do

data/lib/karafka/contracts/consumer_group.rb

@@ -3,9 +3,7 @@
 module Karafka
   module Contracts
     # Contract for single full route (consumer group + topics) validation.
-    class ConsumerGroup < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ConsumerGroup < Base
       # Internal contract for sub-validating topics schema
       TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
 

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -3,9 +3,7 @@
 module Karafka
   module Contracts
     # Consumer group topic validation rules.
-    class ConsumerGroupTopic < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ConsumerGroupTopic < Base
       params do
         required(:consumer).filled
         required(:deserializer).filled

data/lib/karafka/contracts/server_cli_options.rb

@@ -3,9 +3,7 @@
 module Karafka
   module Contracts
     # Contract for validating correctness of the server cli command options.
-    class ServerCliOptions < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
+    class ServerCliOptions < Base
       params do
         optional(:consumer_groups).value(:array, :filled?)
       end

data/lib/karafka/errors.rb

@@ -43,5 +43,9 @@ module Karafka
 
     # Raised when the license token is not valid
     InvalidLicenseTokenError = Class.new(BaseError)
+
+    # Used to instrument this error into the error notifications
+    # We do not raise it so we won't crash deployed systems
+    ExpiredLicenseTokenError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -88,6 +88,9 @@ module Karafka
         when 'connection.listener.fetch_loop.error'
           error "Listener fetch loop error: #{error}"
           error details
+        when 'licenser.expired'
+          error error
+          error details
         when 'runner.call.error'
           fatal "Runner crashed due to an error: #{error}"
           fatal details

data/lib/karafka/licenser.rb

@@ -26,7 +26,7 @@ module Karafka
         data = nil
       end
 
-      details = data ? JSON.parse(data) : raise_invalid_license_token
+      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
 
       license_config.entity = details.fetch('entity')
       license_config.expires_on = Date.parse(details.fetch('expires_on'))
@@ -39,7 +39,11 @@ module Karafka
     private
 
     # Raises an error with info, that used token is invalid
-    def raise_invalid_license_token
+    # @param license_config [Dry::Configurable::Config]
+    def raise_invalid_license_token(license_config)
+      # We set it to false so `Karafka.pro?` method behaves as expected
+      license_config.token = false
+
       raise(
         Errors::InvalidLicenseTokenError,
         <<~MSG.tr("\n", ' ')
@@ -50,15 +54,22 @@ module Karafka
     end
 
     # We do not raise an error here as we don't want to cause any problems to someone that runs
-    # Karafka on production. Error is enough.
+    # Karafka on production. Error message is enough.
+    #
     # @param expires_on [Date] when the license expires
     def notify_if_license_expired(expires_on)
-      Karafka.logger.error(
-        <<~MSG.tr("\n", ' ')
-          Your license expired on #{expires_on}.
-          Karafka no longer uses any Pro capabilities.
-          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-        MSG
+      message = <<~MSG.tr("\n", ' ')
+        Your license expired on #{expires_on}.
+        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+      MSG
+
+      Karafka.logger.error(message)
+
+      Karafka.monitor.instrument(
+        'error.occurred',
+        caller: self,
+        error: Errors::ExpiredLicenseTokenError.new(message),
+        type: 'licenser.expired'
       )
     end
   end

data/lib/karafka/messages/batch_metadata.rb

@@ -14,6 +14,8 @@ module Karafka
       :partition,
       :topic,
       :scheduled_at,
+      :consumption_lag,
+      :processing_lag,
       keyword_init: true
     )
   end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -12,7 +12,12 @@ module Karafka
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
+          # @note Regarding the time lags: we can use the current time here, as batch metadata is
+          #   created in the worker. So whenever this is being built, it means that the processing
+          #   of this batch has already started.
           def call(kafka_batch, topic, scheduled_at)
+            now = Time.now
+
             Karafka::Messages::BatchMetadata.new(
               size: kafka_batch.count,
               first_offset: kafka_batch.first.offset,
@@ -20,9 +25,26 @@ module Karafka
               deserializer: topic.deserializer,
               partition: kafka_batch[0].partition,
               topic: topic.name,
-              scheduled_at: scheduled_at
+              scheduled_at: scheduled_at,
+              # This lag describes how long did it take for a message to be consumed from the
+              # moment it was created
+              consumption_lag: time_distance_in_ms(now, kafka_batch.last.timestamp),
+              # This lag describes how long did a batch have to wait before it was picked up by
+              # one of the workers
+              processing_lag: time_distance_in_ms(now, scheduled_at)
             ).freeze
           end
+
+          private
+
+          # Computes time distance in between two times in ms
+          #
+          # @param time1 [Time]
+          # @param time2 [Time]
+          # @return [Integer] distance in between two times in ms
+          def time_distance_in_ms(time1, time2)
+            ((time1 - time2) * 1_000).round
+          end
         end
       end
     end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Karafka Pro ActiveJob components
+    module ActiveJob
+      # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+      # and that allows to inject additional options into the producer, effectively allowing for a
+      # much better and more granular control over the dispatch and consumption process.
+      class Dispatcher < ::Karafka::ActiveJob::Dispatcher
+        # Defaults for dispatching
+        # The can be updated by using `#karafka_options` on the job
+        DEFAULTS = {
+          dispatch_method: :produce_async,
+          # We don't create a dummy proc based partitioner as we would have to evaluate it with
+          # each job.
+          partitioner: nil
+        }.freeze
+
+        private_constant :DEFAULTS
+
+        # @param job [ActiveJob::Base] job
+        def call(job)
+          ::Karafka.producer.public_send(
+            fetch_option(job, :dispatch_method, DEFAULTS),
+            dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(job.serialize)
+            )
+          )
+        end
+
+        private
+
+        # @param job [ActiveJob::Base] job instance
+        # @return [Hash] hash with dispatch details to which we merge topic and payload
+        def dispatch_details(job)
+          partitioner = fetch_option(job, :partitioner, DEFAULTS)
+
+          return {} unless partitioner
+
+          {
+            partition_key: partitioner.call(job)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # Contract for validating the options that can be altered with `#karafka_options` per job
+      # class that works with Pro features.
+      class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract
+        # Dry types
+        Types = include Dry.Types()
+
+        params do
+          optional(:partitioner).value(Types.Interface(:call))
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+module Karafka
+  module Pro
+    # Loader requires and loads all the pro components only when they are needed
+    class Loader
+      class << self
+        # Loads all the pro components and configures them wherever it is expected
+        # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
+        #   components
+        def setup(config)
+          require_relative 'active_job/dispatcher'
+          require_relative 'active_job/job_options_contract'
+
+          config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
+          config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+module Karafka
+  # Namespace for pro components, licensed under the commercial license agreement.
+  module Pro
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -41,7 +41,7 @@ module Karafka
       # We signal critical exceptions, notify and do not allow worker to fail
       # rubocop:disable Lint/RescueException
       rescue Exception => e
-      # rubocop:enable Lint/RescueException
+        # rubocop:enable Lint/RescueException
         Karafka.monitor.instrument(
           'error.occurred',
           caller: self,

data/lib/karafka/railtie.rb

@@ -19,6 +19,7 @@ end
 if rails
   # Load Karafka
   require 'karafka'
+
   # Load ActiveJob adapter
   require 'active_job/karafka'
 
@@ -30,6 +31,13 @@ if rails
     class Railtie < Rails::Railtie
       railtie_name :karafka
 
+      initializer 'karafka.active_job_integration' do
+        ActiveSupport.on_load(:active_job) do
+          # Extend ActiveJob with some Karafka specific ActiveJob magic
+          extend ::Karafka::ActiveJob::JobExtensions
+        end
+      end
+
       initializer 'karafka.configure_rails_initialization' do |app|
         # Consumers should autoload by default in the Rails app so they are visible
         app.config.autoload_paths += %w[app/consumers]
@@ -37,8 +45,10 @@ if rails
         # Make Karafka use Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
-        # This lines will make Karafka print to stdout like puma or unicorn
-        if Rails.env.development?
+        # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+        # server + will support code reloading with each fetched loop. We do it only for karafka
+        # based commands as Rails processes and console will have it enabled already
+        if Rails.env.development? && ENV.key?('KARAFKA_CLI')
           Rails.logger.extend(
             ActiveSupport::Logger.broadcast(
               ActiveSupport::Logger.new($stdout)

data/lib/karafka/routing/builder.rb

@@ -10,11 +10,6 @@ module Karafka
     #     end
     #   end
     class Builder < Concurrent::Array
-      # Consumer group consistency checking contract
-      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
-
-      private_constant :CONTRACT
-
       def initialize
         @draws = Concurrent::Array.new
         super
@@ -38,12 +33,7 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
-          hashed_group = consumer_group.to_h
-          validation_result = CONTRACT.call(hashed_group)
-
-          next if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
         end
       end
 

data/lib/karafka/routing/subscription_group.rb

@@ -20,7 +20,7 @@ module Karafka
 
       # @return [String] consumer group id
       def consumer_group_id
-        kafka['group.id']
+        kafka[:'group.id']
       end
 
       # @return [Integer] max messages fetched in a single go
@@ -39,12 +39,12 @@ module Karafka
       def kafka
         kafka = @topics.first.kafka.dup
 
-        kafka['client.id'] ||= Karafka::App.config.client_id
-        kafka['group.id'] ||= @topics.first.consumer_group.id
-        kafka['auto.offset.reset'] ||= 'earliest'
+        kafka[:'client.id'] ||= Karafka::App.config.client_id
+        kafka[:'group.id'] ||= @topics.first.consumer_group.id
+        kafka[:'auto.offset.reset'] ||= 'earliest'
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka['enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = 'false'
         kafka.freeze
         kafka
       end

data/lib/karafka/setup/config.rb

@@ -14,15 +14,12 @@ module Karafka
     class Config
       extend Dry::Configurable
 
-      # Contract for checking the config provided by the user
-      CONTRACT = Karafka::Contracts::Config.new.freeze
-
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id' => 'karafka'
+        'client.id': 'karafka'
       }.freeze
 
-      private_constant :CONTRACT, :KAFKA_DEFAULTS
+      private_constant :KAFKA_DEFAULTS
 
       # Available settings
 
@@ -84,8 +81,6 @@ module Karafka
       setting :kafka, default: {}
 
       # Namespace for internal settings that should not be modified
-      # It's a temporary step to "declassify" several things internally before we move to a
-      # non global state
       setting :internal do
         # option routing_builder [Karafka::Routing::Builder] builder instance
         setting :routing_builder, default: Routing::Builder.new
@@ -98,6 +93,17 @@ module Karafka
         # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
         #   group builder
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+        # Karafka components for ActiveJob
+        setting :active_job do
+          # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
+          setting :dispatcher, default: ActiveJob::Dispatcher.new
+          # option job_options_contract [Karafka::Contracts::JobOptionsContract] contract for
+          #   ensuring, that extra job options defined are valid
+          setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
+          # option consumer [Class] consumer class that should be used to consume ActiveJob data
+          setting :consumer, default: ActiveJob::Consumer
+        end
       end
 
       class << self
@@ -106,7 +112,7 @@ module Karafka
         def setup(&block)
           configure(&block)
           merge_kafka_defaults!(config)
-          validate!
+          Contracts::Config.new.validate!(config.to_h)
 
           # Check the license presence (if needed) and
           Licenser.new.verify(config.license)
@@ -128,18 +134,6 @@ module Karafka
           end
         end
 
-        # Validate config based on the config contract
-        # @return [Boolean] true if configuration is valid
-        # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-        #   doesn't match with the config contract
-        def validate!
-          validation_result = CONTRACT.call(config.to_h)
-
-          return true if validation_result.success?
-
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
-        end
-
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components
@@ -149,6 +143,12 @@ module Karafka
             producer_config.kafka = config.kafka.dup
             producer_config.logger = config.logger
           end
+
+          return unless Karafka.pro?
+
+          # Runs the pro loader that includes all the pro components
+          require 'karafka/pro/loader'
+          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha1'
+  VERSION = '2.0.0.alpha2'
 end

data/lib/karafka.rb

@@ -65,6 +65,11 @@ module Karafka
       Pathname.new(File.expand_path('karafka', __dir__))
     end
 
+    # @return [Boolean] true if there is a valid pro token present
+    def pro?
+      App.config.license.token != false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
@@ -82,8 +87,8 @@ end
 
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
-loader.do_not_eager_load(Karafka.gem_root.join('lib/active_job'))
+loader.ignore(Karafka.gem_root.join('lib/active_job'))
 # Do not load pro components, this will be handled by license manager
-loader.do_not_eager_load(Karafka.gem_root.join('lib/karafka/pro'))
+loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
 loader.setup
 loader.eager_load

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha1
+  version: 2.0.0.alpha2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-01-30 00:00:00.000000000 Z
+date: 2022-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -112,7 +112,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -178,11 +178,14 @@ files:
 - config/errors.yml
 - docker-compose.yml
 - karafka.gemspec
-- lib/active_job/consumer.rb
 - lib/active_job/karafka.rb
 - lib/active_job/queue_adapters/karafka_adapter.rb
-- lib/active_job/routing_extensions.rb
 - lib/karafka.rb
+- lib/karafka/active_job/consumer.rb
+- lib/karafka/active_job/dispatcher.rb
+- lib/karafka/active_job/job_extensions.rb
+- lib/karafka/active_job/job_options_contract.rb
+- lib/karafka/active_job/routing_extensions.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -197,6 +200,7 @@ files:
 - lib/karafka/connection/pauses_manager.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
+- lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
 - lib/karafka/contracts/consumer_group_topic.rb
@@ -221,6 +225,10 @@ files:
 - lib/karafka/messages/metadata.rb
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/consumer.rb
+- lib/karafka/pro.rb
+- lib/karafka/pro/active_job/dispatcher.rb
+- lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/loader.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
@@ -275,7 +283,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/active_job/consumer.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module ActiveJob
-  # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
-  # It marks the offset after each message, so we make sure, none of the jobs is executed twice
-  class Consumer < Karafka::BaseConsumer
-    # Executes the ActiveJob logic
-    # @note ActiveJob does not support batches, so we just run one message after another
-    def consume
-      messages.each do |message|
-        ActiveJob::Base.execute(
-          # We technically speaking could set this as deserializer and reference it from the
-          # message instead of using the `#raw_payload`. This is not done on purpose to simplify
-          # the ActiveJob setup here
-          ActiveSupport::JSON.decode(message.raw_payload)
-        )
-
-        mark_as_consumed(message)
-      end
-    end
-  end
-end

data/lib/active_job/routing_extensions.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module ActiveJob
-  # Routing extensions for ActiveJob
-  module RoutingExtensions
-    # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting the
-    # consumer class
-    # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-    def active_job_topic(name)
-      topic(name) do
-        consumer ActiveJob::Consumer
-      end
-    end
-  end
-end