karafka 2.0.0 → 2.1.0

data/Gemfile

@@ -6,10 +6,11 @@ plugin 'diffend'
 
 gemspec
 
-# Karafka gem does not require this but we add it here so we can test the integration with
-# ActiveJob much easier
+# Karafka gem does not require activejob nor karafka-web  to work
+# They are added here because they are part of the integration suite
 group :integrations do
   gem 'activejob'
+  gem 'karafka-web'
 end
 
 group :test do

data/Gemfile.lock

@@ -1,73 +1,84 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0)
-      karafka-core (>= 2.0.2, < 3.0.0)
-      rdkafka (>= 0.12)
+    karafka (2.1.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.1, < 3.0.0)
+      waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.3.1)
-      activesupport (= 7.0.3.1)
+    activejob (7.0.4.3)
+      activesupport (= 7.0.4.3)
       globalid (>= 0.3.6)
-    activesupport (7.0.3.1)
+    activesupport (7.0.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    erubi (1.12.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    globalid (1.0.0)
+    globalid (1.1.0)
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.2)
+    karafka-core (2.0.12)
       concurrent-ruby (>= 1.1)
-    mini_portile2 (2.8.0)
-    minitest (5.16.2)
-    rake (13.0.6)
-    rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12.1)
+    karafka-rdkafka (0.12.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    karafka-web (0.5.1)
+      erubi (~> 1.4)
+      karafka (>= 2.0.40, < 3.0.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
+      roda (~> 3.63)
+      tilt (~> 2.0)
+    mini_portile2 (2.8.1)
+    minitest (5.18.0)
+    rack (3.0.7)
+    rake (13.0.6)
+    roda (3.67.0)
+      rack
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
-    simplecov (0.21.2)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tzinfo (2.0.5)
+    tilt (2.1.0)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.1)
-      karafka-core (>= 2.0.2, < 3.0.0)
-      rdkafka (>= 0.10)
+    waterdrop (2.5.1)
+      karafka-core (>= 2.0.12, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.0)
+    zeitwerk (2.6.7)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -75,8 +86,9 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
+  karafka-web
   rspec
   simplecov
 
 BUNDLED WITH
-   2.3.15
+   2.4.10

data/LICENSE

@@ -12,6 +12,6 @@ and their usage requires commercial license agreement.
 By sending a pull request to the pro components, you are agreeing to transfer the copyright of your
 code to Maciej Mensfeld.
 
-You can find the commercial license in LICENSE-COM.
+You can find the commercial license in LICENSE-COMM.
 
 Please see https://karafka.io for purchasing options.

data/README.md

@@ -4,18 +4,20 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
+**Note**: Upgrade instructions for migration from Karafka `1.4` to Karafka `2.0` can be found [here](https://karafka.io/docs/Upgrades-2.0/).
 
 ## About Karafka
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
-- Supports parallel processing in [multiple threads](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading) (also for a [single topic partition](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions) work)
-- Has [ActiveJob backend](https://github.com/karafka/karafka/wiki/Active-Job) support (including [ordered jobs](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Active-Job#ordered-jobs))
-- [Automatically integrates](https://github.com/karafka/karafka/wiki/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails=) with Ruby on Rails
-- Supports in-development [code reloading](https://github.com/karafka/karafka/wiki/Auto-reload-of-code-changes-in-development)
+- Has a built-in [Web UI](https://karafka.io/docs/Web-UI-Features/) providing a convenient way to monitor and manage Karafka-based applications.
+- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
+- [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
+- Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
+- Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
+- Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
-- Has an out-of the box [StatsD/DataDog monitoring](https://github.com/karafka/karafka/wiki/Monitoring-and-logging) with a dashboard template.
+- Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb
@@ -39,23 +41,26 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
 
 ## Getting started
 
+![karafka web ui](https://raw.githubusercontent.com/karafka/misc/master/printscreens/web-ui.png)
+
 If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
 
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Rails and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
 
-If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to visit our [Getting started](https://github.com/karafka/karafka/wiki/Getting-started) guides and the [example apps repository](https://github.com/karafka/example-apps).
+If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to visit our [Getting started](https://karafka.io/docs/Getting-Started) guides and the [example apps repository](https://github.com/karafka/example-apps).
 
 We also maintain many [integration specs](https://github.com/karafka/karafka/tree/master/spec/integrations) illustrating various use-cases and features of the framework.
 
 ### TL;DR (1 minute from setup to publishing and consuming messages)
 
-**Prerequisites**: Kafka running. You can start it by following instructions from [here](https://github.com/karafka/karafka/wiki/Setting-up-Kafka).
+**Prerequisites**: Kafka running. You can start it by following instructions from [here](https://karafka.io/docs/Setting-up-Kafka).
 
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka
+# Make sure to install Karafka 2.0
+bundle add karafka --version ">= 2.0.28"
 
 bundle exec karafka install
 ```
@@ -71,24 +76,24 @@ Karafka.producer.produce_sync(topic: 'example', payload: { 'ping' => 'pong' }.to
 ```bash
 bundle exec karafka server
 
-[7616dc24-505a-417f-b87b-6bf8fc2d98c5] Polled 1 message in 1000ms
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example started
+[86d47f0b92f7] Polled 1 message in 1000ms
+[3732873c8a74] Consume job for ExampleConsumer on example started
 {"ping"=>"pong"}
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example finished in 0ms
+[3732873c8a74] Consume job for ExampleConsumer on example finished in 0ms
 ```
 
 ## Want to Upgrade? LGPL is not for you? Want to help?
 
-I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
+I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, enhanced Web UI and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
 
-**20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
+**10%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 
-Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io) for more details.
+Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io/#become-pro) for more details.
 
 ## Support
 
-Karafka has [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the installation, setup, and deployment, along with other useful details on how to run Karafka.
+Karafka has [Wiki pages](https://karafka.io/docs) for almost everything and a pretty decent [FAQ](https://karafka.io/docs/FAQ). It covers the installation, setup, and deployment, along with other useful details on how to run Karafka.
 
 If you have questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) channel.
 
-Karafka has priority support for technical and architectural questions that is part of the Karafka Pro subscription.
+Karafka has [priority support](https://karafka.io/docs/Pro-Support) for technical and architectural questions that is part of the Karafka Pro subscription.

data/bin/integrations

@@ -2,9 +2,14 @@
 
 # Runner to run integration specs in parallel
 
-# Part of integration specs run pristine without bundler.
+# Part of integration specs run linear without bundler.
 # If we would run bundle exec when running this code, bundler would inject its own context
 # into them, messing things up heavily
+#
+# Types of specs:
+# - regular - can run in parallel, includes all the helpers
+# - pristine - cannot run in parallel, uses custom bundler but includes helpers
+# - poro - cannot run in parallel, uses custom bundler, does not include any helpers
 raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
 
 require 'open3'
@@ -16,10 +21,11 @@ require 'etc'
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
 # How many child processes with integration specs do we want to run in parallel
-# When the value is high, there's a problem with thread allocation on Github CI, tht is why
+# When the value is high, there's a problem with thread allocation on Github CI, that is why
 # we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
-# of CPU
-CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
+# of CPU. Locally we also cannot go beyond certain limit due to how often and how many topics we
+# create in Kafka. With an overloaded system, we start getting timeouts.
+CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
 MAX_BUFFER_OUTPUT = 51_200
@@ -28,16 +34,16 @@ MAX_BUFFER_OUTPUT = 51_200
 class Scenario
   # How long a scenario can run before we kill it
   # This is a fail-safe just in case something would hang
-  MAX_RUN_TIME = 3 * 60 # 3 minutes tops
+  MAX_RUN_TIME = 5 * 60 # 5 minutes tops
 
   # There are rare cases where Karafka may force shutdown for some of the integration cases
   # This includes exactly those
   EXIT_CODES = {
     default: [0],
-    'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
-    'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
+    'consumption/worker_critical_error_behaviour_spec.rb' => [0, 2].freeze,
+    'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
+    'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
+    'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES
@@ -47,6 +53,8 @@ class Scenario
   # @param path [String] path to the scenarios file
   def initialize(path)
     @path = path
+    # First 1024 characters from stdout
+    @stdout_head = ''
     # Last 1024 characters from stdout
     @stdout_tail = ''
   end
@@ -62,21 +70,25 @@ class Scenario
     @path.gsub("#{ROOT_PATH}/spec/integrations/", '')
   end
 
-  # @return [Boolean] true if spec is pristine
-  def pristine?
+  # @return [Symbol] type of spec
+  def type
     scenario_dir = File.dirname(@path)
 
-    # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
-    # to run bundle install, etc in order to run it
-    File.exist?(File.join(scenario_dir, 'Gemfile'))
+    return :poro if scenario_dir.include?('_poro')
+    return :pristine if scenario_dir.include?('_pristine')
+
+    :regular
+  end
+
+  # @return [Boolean] any spec that is not a regular one should not run in parallel with others
+  def linear?
+    type != :regular
   end
 
   # @return [Boolean] did this scenario finished or is it still running
   def finished?
     # If the thread is running too long, kill it
     if current_time - @started_at > MAX_RUN_TIME
-      @wait_thr.kill
-
       begin
         Process.kill('TERM', pid)
       # It may finish right after we want to kill it, that's why we ignore this
@@ -88,6 +100,7 @@ class Scenario
     # to stdout. Otherwise after reaching the buffer size, it would hang
     buffer = ''
     @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
+    @stdout_head = buffer if @stdout_head.empty?
     @stdout_tail << buffer
     @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
 
@@ -112,6 +125,11 @@ class Scenario
     @wait_thr.value&.exitstatus || 123
   end
 
+  # @return [String] exit status of the process
+  def exit_status
+    @wait_thr.value.to_s
+  end
+
   # Prints a status report when scenario is finished and stdout if it failed
   def report
     if success?
@@ -123,21 +141,52 @@ class Scenario
 
       puts
       puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
+      puts "Time taken: #{current_time - @started_at} seconds"
       puts "Exit code: #{exit_code}"
+      puts "Exit status: #{exit_status}"
+      puts @stdout_head
+      puts '...'
       puts @stdout_tail
       puts buffer
       puts
     end
   end
 
+  # @return [Float] number of seconds that a given spec took to run
+  def time_taken
+    @finished_at - @started_at
+  end
+
+  # Close all the files that are open, so they do not pile up
+  def close
+    @finished_at = current_time
+    @stdin.close
+    @stdout.close
+    @stderr.close
+  end
+
   private
 
   # Sets up a proper environment for a given spec to run and returns the run command
   # @return [String] run command
   def init_and_build_cmd
-    # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
-    # to run bundle install, etc in order to run it
-    if pristine?
+    case type
+    when :poro
+      scenario_dir = File.dirname(@path)
+      # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
+      temp_dir = Dir.mktmpdir
+      file_name = File.basename(@path)
+
+      FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
+
+      <<~CMD
+        cd #{temp_dir} &&
+        KARAFKA_GEM_DIR=#{ROOT_PATH} \
+        BUNDLE_AUTO_INSTALL=true \
+        PRISTINE_MODE=true \
+        bundle exec ruby #{file_name}
+      CMD
+    when :pristine
       scenario_dir = File.dirname(@path)
       # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
       temp_dir = Dir.mktmpdir
@@ -167,7 +216,7 @@ class Scenario
 end
 
 # Load all the specs
-specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
+specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
 
 # If filters is provided, apply
 # Allows to provide several filters one after another and applies all of them
@@ -175,6 +224,7 @@ ARGV.each do |filter|
   specs.delete_if { |name| !name.include?(filter) }
 end
 
+
 raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
 
 # Randomize order
@@ -186,8 +236,8 @@ scenarios = specs
             .shuffle(random: Random.new(seed))
             .map { |integration_test| Scenario.new(integration_test) }
 
-regulars = scenarios.reject(&:pristine?)
-pristine = scenarios.select(&:pristine?)
+regulars = scenarios.reject(&:linear?)
+linears = scenarios - regulars
 
 active_scenarios = []
 finished_scenarios = []
@@ -196,10 +246,10 @@ while finished_scenarios.size < scenarios.size
   # If we have space to run another scenario, we add it
   if active_scenarios.size < CONCURRENCY
     scenario = nil
-    # We can run only one pristine at the same time due to concurrency issues within bundler
+    # We can run only one linear at the same time due to concurrency issues within bundler
     # Since they usually take longer than others, we try to run them as fast as possible when there
     # is a slot
-    scenario = pristine.pop unless active_scenarios.any?(&:pristine?)
+    scenario = linears.pop unless active_scenarios.any?(&:linear?)
     scenario ||= regulars.pop
 
     if scenario
@@ -211,19 +261,29 @@ while finished_scenarios.size < scenarios.size
   active_scenarios.select(&:finished?).each do |exited|
     scenario = active_scenarios.delete(exited)
     scenario.report
+    scenario.close
     finished_scenarios << scenario
   end
 
   sleep(0.1)
 end
 
+# Report longest scenarios
+puts
+puts "\nLongest scenarios:\n\n"
+
+finished_scenarios.sort_by(&:time_taken).reverse.first(10).each do |long_scenario|
+  puts "[#{'%6.2f' % long_scenario.time_taken}] #{long_scenario.name}"
+end
+
 failed_scenarios = finished_scenarios.reject(&:success?)
 
-# Report once more on the failed jobs
-# This will only list scenarios that failed without printing their stdout here.
 if failed_scenarios.empty?
   puts
 else
+  # Report once more on the failed jobs
+  # This will only list scenarios that failed without printing their stdout here.
+  puts
   puts "\nFailed scenarios:\n\n"
 
   failed_scenarios.each do |scenario|

data/bin/karafka

@@ -6,18 +6,6 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-# If there is a boot file, we need to require it as we expect it to contain
-# Karafka app setup, routes, etc
-if File.exist?(Karafka.boot_file)
-  require Karafka.boot_file.to_s
-else
-  # However when it is unavailable, we still want to be able to run help command
-  # and install command as they don't require configured app itself to run
-  raise(
-    Karafka::Errors::MissingBootFileError,
-    Karafka.boot_file
-  ) unless %w[-h install].include?(ARGV[0])
-end
-
+Karafka::Cli::Base.load
 Karafka::Cli.prepare
 Karafka::Cli.start

data/bin/rspecs

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+SPECS_TYPE=regular bundle exec rspec --tag ~type:pro
+SPECS_TYPE=pro bundle exec rspec --tag type:pro

data/bin/verify_license_integrity

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This script verifies integrity of the Pro license
+# Run it before bundle install to ensure, that what you are fetching is what you expect
+# Run it after bundle install to ensure that the local artefact was not compromised
+
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$MODE" != "after" ]; then
+  # Check the remote license prior to bundle installing
+  curl \
+    --retry 5 \
+    --retry-delay 1 \
+    --fail \
+    -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
+    https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \
+    -o ./karafka-license.gem
+else
+  # Check the local cached one after bundle install
+  cache_path=`ruby -e 'puts "#{Gem.dir}/cache/"'`
+  cp "$cache_path/karafka-license-$KARAFKA_PRO_VERSION.gem" ./karafka-license.gem
+fi
+
+detected=`sha256sum ./karafka-license.gem | awk '{ print $1 }'`
+
+rm ./karafka-license.gem
+
+echo -n "Karafka Pro license artifact checksum verification result: "
+
+if [ "$detected" = "$KARAFKA_PRO_LICENSE_CHECKSUM" ]; then
+  echo "Success"
+else
+  echo -e "\033[0;31mFailure!\033[0m"
+  exit 1
+fi

data/certs/cert_chain.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
+jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
+F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
+ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
+lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
+rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
+KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
+Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
+cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
+iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
+G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
+vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
+yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
+EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
+Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
+KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
+Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
+MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+-----END CERTIFICATE-----

data/config/{errors.yml → locales/errors.yml}

@@ -15,6 +15,13 @@ en:
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
       kafka_format: needs to be a filled hash
+      internal.processing.jobs_builder_format: cannot be nil
+      internal.processing.scheduler: cannot be nil
+      internal.processing.coordinator_class: cannot be nil
+      internal.processing.partitioner_class: cannot be nil
+      internal.active_job.dispatcher: cannot be nil
+      internal.active_job.job_options_contract: cannot be nil
+      internal.active_job.consumer_class: cannot be nil
       internal.status_format: needs to be present
       internal.process_format: needs to be present
       internal.routing.builder_format: needs to be present
@@ -25,32 +32,53 @@ en:
 
     server_cli_options:
       missing: needs to be present
-      consumer_groups_inclusion: Unknown consumer group
+      consumer_groups_inclusion: Unknown consumer group name
+      subscription_groups_inclusion: Unknown subscription group name
+      topics_inclusion: Unknown topic name
+      topics_missing: No topics to subscribe to
 
-    consumer_group_topic:
+    topic:
+      kafka: needs to be a hash with kafka scope settings details
       missing: needs to be present
+      max_messages_format: 'needs to be an integer bigger than 0'
+      max_wait_time_format: 'needs to be an integer bigger than 0'
       name_format: 'needs to be a string with a Kafka accepted format'
       deserializer_format: needs to be present
-      manual_offset_management_format: needs to be either true or false
       consumer_format: needs to be present
       id_format: 'needs to be a string with a Kafka accepted format'
       initial_offset_format: needs to be either earliest or latest
+      subscription_group_format: must be a non-empty string
+      manual_offset_management.active_format: needs to be either true or false
+      consumer_active_job_missing: ActiveJob needs to be available
+      manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
+      dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
+      dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
+      dead_letter_queue.active_format: needs to be either true or false
+      active_format: needs to be either true or false
+      declaratives.partitions_format: needs to be more or equal to 1
+      declaratives.active_format: needs to be true
+      declaratives.replication_factor_format: needs to be more or equal to 1
+      declaratives.details_format: needs to be a hash with only symbol keys
+      inconsistent_namespacing: |
+        needs to be consistent namespacing style
+        disable this validation by setting config.strict_topics_namespacing to false
 
     consumer_group:
       missing: needs to be present
       topics_names_not_unique: all topic names within a single consumer group must be unique
       id_format: 'needs to be a string with a Kafka accepted format'
       topics_format: needs to be a non-empty array
+      topics_namespaced_names_not_unique: |
+        all topic names within a single consumer group must be unique considering namespacing styles
+        disable this validation by setting config.strict_topics_namespacing to false
 
     job_options:
       missing: needs to be present
       dispatch_method_format: needs to be either :produce_async or :produce_sync
+      dispatch_many_method_format: needs to be either :produce_many_async or :produce_many_sync
       partitioner_format: 'needs to respond to #call'
       partition_key_type_format: 'needs to be either :key or :partition_key'
 
     test:
       missing: needs to be present
       id_format: needs to be a String
-
-    pro_consumer_group_topic:
-      consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer