karafka 2.5.0.beta1 → 2.5.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bcac2dd9a093cd85ab76342d9c227d552e9927e681e79f55963a9d13e64f79d
-  data.tar.gz: 8b5e32c1c1099b8e654599c6df25c33816c3a4a315ffe7e0de06c49a2b4e720e
+  metadata.gz: bbcaf396d7f2eff35ec3b59c96ffe7dd880f1f07294aa28bb623e95ce328e3e9
+  data.tar.gz: 410b037a79abbbc82fbd4b540dcb26378dca396e408f68280325e354ae2e275b
 SHA512:
-  metadata.gz: d2ffa6709d42103eea487e44b6756d9b449bd762b162e8ce35283b2513b26c2fde8ef559b900c7932d1fe4e3e8c771d9874d1ed54a572e3e94f6df067eb354d8
-  data.tar.gz: 9bfafb2b2c8ec8975e55fe1bddeaaedfb4d7fd1cedae36abdfed45819e8136eb614922d4f52c0b011d97602ad728dc7a860a3025e9ab1694720b0f35af00429d
+  metadata.gz: dcd79f3bda653b74d95938d440ccfe93d8d2cdc214e02de31d9ad56f738b554699bd1991d83dd1c9c7090e1c3a63669f8837677fef77efc22df4804bc16f25a0
+  data.tar.gz: dbca53f433a0e13ec7582f9ee1c6da15bf38678a1019a3a36e361e7bbbae2c5e5777bf44e61330692b5f38c4fa011dd9e3950027925767908f617df02e1e8219

data/.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
+        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
         with:
           ruby-version: 3.4
           bundler-cache: true
@@ -118,7 +118,7 @@ jobs:
         run: rm -f Gemfile.lock
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
+        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -164,7 +164,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
+        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
         with:
           # Do not use cache here as we run bundle install also later in some of the integration
           # tests and we need to be able to run it without cache
@@ -228,7 +228,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0
+        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler: 'latest'

data/.github/workflows/push.yml

@@ -0,0 +1,36 @@
+name: Push Gem
+
+on:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+jobs:
+  push:
+    if: github.repository_owner == 'karafka'
+    runs-on: ubuntu-latest
+    environment: deployment
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        with:
+          bundler-cache: false
+
+      - name: Bundle install
+        run: |
+          bundle install --jobs 4 --retry 3
+
+      # Release
+      - uses: rubygems/release-gem@9e85cb11501bebc2ae661c1500176316d3987059 # v1

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka Framework Changelog
 
-## 2.4.19 (Unreleased)
+## 2.5.0 (Unreleased)
 - **[Breaking]** Use DLQ and Piping prefix `source_` instead of `original_` to align with naming convention of Kafka Streams and Apache Flink for future usage.
 - **[Breaking]** Rename scheduled jobs topics names in their config (Pro).
 - **[Feature]** Parallel Segments for concurrent processing of the same partition with more than partition count of processes (Pro).
@@ -30,6 +30,11 @@
 - [Enhancement] Set `topic.metadata.refresh.interval.ms` for default producer in dev to 5s to align with consumer setup.
 - [Enhancement] Alias `-2` and `-1` with `latest` and `earliest` for seeking.
 - [Enhancement] Allow for usage of `latest` and `earliest` in the `Karafka::Pro::Iterator`.
+- [Enhancement] Failures during `topics migrate` (and other subcommands) don't show what topic failed, and why it's invalid.
+- [Enhancement] Apply changes to topics configuration in atomic independent requests when using Declarative Topics.
+- [Enhancement] Execute the help CLI command when no command provided (similar to Rails) to improve DX.
+- [Enhancement] Remove backtrace from the CLI error for incorrect commands (similar to Rails) to improve DX.
+- [Enhancement] Provide `karafka topics help` sub-help due to nesting of Declarative Topics actions.
 - [Refactor] Introduce a `bin/verify_kafka_warnings` script to clean Kafka from temporary test-suite topics.
 - [Refactor] Introduce a `bin/verify_topics_naming` script to ensure proper test topics naming convention.
 - [Refactor] Make sure all temporary topics have a `it-` prefix in their name.
@@ -52,6 +57,8 @@
 - [Fix] optparse double parse loses ARGV.
 - [Fix] `karafka` cannot be required without Bundler.
 - [Fix] Scheduled Messages re-seek moves to `latest` on inheritance of initial offset when `0` offset is compacted.
+- [Fix] Seek to `:latest` without `topic_partition_position` (-1) will not seek at all.
+- [Change] Move to trusted-publishers and remove signing since no longer needed.
 
 ## 2.4.18 (2025-04-09)
 - [Fix] Make sure `Bundler.with_unbundled_env` is not called multiple times.

data/Gemfile

@@ -16,9 +16,9 @@ group :integrations, :test do
 end
 
 group :integrations do
-  # gem 'activejob', require: false
-  # gem 'karafka-testing', '>= 2.4.6', require: false
-  # gem 'karafka-web', '>= 0.10.4', require: false
+  gem 'activejob', require: false
+  gem 'karafka-testing', '>= 2.5.0', require: false
+  gem 'karafka-web', '>= 0.11.0.beta1', require: false
 end
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.5.0.beta1)
+    karafka (2.5.0.beta2)
       base64 (~> 0.2)
       karafka-core (>= 2.5.0, < 2.6.0)
       karafka-rdkafka (>= 0.19.2)
@@ -11,6 +11,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    activejob (8.0.2)
+      activesupport (= 8.0.2)
+      globalid (>= 0.3.6)
     activesupport (8.0.2)
       base64
       benchmark (>= 0.3)
@@ -33,6 +36,7 @@ GEM
     diff-lcs (1.6.2)
     docile (1.4.1)
     drb (2.2.3)
+    erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
     factory_bot (6.5.1)
@@ -51,21 +55,35 @@ GEM
     fugit (1.11.1)
       et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.5.0)
+    karafka-core (2.5.1)
       karafka-rdkafka (>= 0.19.2, < 0.21.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.19.2)
+    karafka-rdkafka (0.19.4)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    karafka-testing (2.5.0)
+      karafka (>= 2.5.0.beta1, < 2.6.0)
+      waterdrop (>= 2.8.0)
+    karafka-web (0.11.0.beta3)
+      erubi (~> 1.4)
+      karafka (>= 2.5.0.beta1, < 2.6.0)
+      karafka-core (>= 2.5.0, < 2.6.0)
+      roda (~> 3.68, >= 3.69)
+      tilt (~> 2.0)
     logger (1.7.0)
     mini_portile2 (2.8.9)
     minitest (5.25.5)
     ostruct (0.6.1)
     raabro (1.4.0)
+    rack (3.1.15)
     rake (13.2.1)
+    roda (3.92.0)
+      rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -87,14 +105,15 @@ GEM
     simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
     stringio (3.1.7)
+    tilt (2.6.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uri (1.0.3)
-    waterdrop (2.8.3)
+    waterdrop (2.8.4)
       karafka-core (>= 2.4.9, < 3.0.0)
-      karafka-rdkafka (>= 0.19.1)
+      karafka-rdkafka (>= 0.19.2)
       zeitwerk (~> 2.3)
-    zeitwerk (2.7.3)
+    zeitwerk (2.6.18)
 
 PLATFORMS
   aarch64-linux-gnu
@@ -110,10 +129,13 @@ PLATFORMS
   x86_64-linux-musl
 
 DEPENDENCIES
+  activejob
   byebug
   factory_bot
   fugit
   karafka!
+  karafka-testing (>= 2.5.0)
+  karafka-web (>= 0.11.0.beta1)
   ostruct
   rspec
   simplecov

data/README.md

@@ -84,7 +84,7 @@ bundle exec karafka server
 
 I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, enhanced Web UI and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
 
-**10%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
+Part of the income is [distributed back](https://github.com/orgs/karafka/sponsoring) to other OSS projects that Karafka uses under the hood.
 
 Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io/#become-pro) for more details.
 

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'bundler/gem_tasks'

data/bin/integrations

@@ -48,7 +48,8 @@ class Scenario
     'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze,
     'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
     'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
-    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze
+    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/align/incorrectly_spec.rb' => [1].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/karafka.gemspec

@@ -29,11 +29,6 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 3.0.0'
 
-  if $PROGRAM_NAME.end_with?('gem')
-    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
-  end
-
-  spec.cert_chain    = %w[certs/cert.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]

data/lib/karafka/admin/configs.rb

@@ -10,6 +10,10 @@ module Karafka
     #
     # Altering is done in the incremental way.
     module Configs
+      extend Helpers::ConfigImporter.new(
+        max_wait_time: %i[admin max_wait_time]
+      )
+
       class << self
         # Fetches given resources configurations from Kafka
         #
@@ -94,7 +98,7 @@ module Karafka
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
           Admin.with_admin do |admin|
-            yield(admin).wait(max_wait_timeout: Karafka::App.config.admin.max_wait_time)
+            yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end
       end

data/lib/karafka/admin.rb

@@ -10,6 +10,15 @@ module Karafka
   #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
   #   there is no multi-cluster runtime support.
   module Admin
+    extend Helpers::ConfigImporter.new(
+      max_wait_time: %i[admin max_wait_time],
+      poll_timeout: %i[admin poll_timeout],
+      max_attempts: %i[admin max_attempts],
+      group_id: %i[admin group_id],
+      app_kafka: %i[kafka],
+      admin_kafka: %i[admin kafka]
+    )
+
     # 2010-01-01 00:00:00 - way before Kafka was released so no messages should exist prior to
     # this date
     # We do not use the explicit -2 librdkafka value here because we resolve this offset without
@@ -113,7 +122,7 @@ module Karafka
           handler = admin.create_topic(name, partitions, replication_factor, topic_config)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { topics_names.include?(name) }
           )
         end
@@ -127,7 +136,7 @@ module Karafka
           handler = admin.delete_topic(name)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { !topics_names.include?(name) }
           )
         end
@@ -142,7 +151,7 @@ module Karafka
           handler = admin.create_partitions(name, partitions)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { topic_info(name).fetch(:partition_count) >= partitions }
           )
         end
@@ -353,7 +362,7 @@ module Karafka
       def delete_consumer_group(consumer_group_id)
         with_admin do |admin|
           handler = admin.delete_group(consumer_group_id)
-          handler.wait(max_wait_timeout: app_config.admin.max_wait_time)
+          handler.wait(max_wait_timeout: max_wait_time)
         end
       end
 
@@ -539,7 +548,7 @@ module Karafka
 
         admin = config(:producer, {}).admin(
           native_kafka_auto_start: false,
-          native_kafka_poll_timeout_ms: app_config.admin.poll_timeout
+          native_kafka_poll_timeout_ms: poll_timeout
         )
 
         bind_oauth(bind_id, admin)
@@ -604,7 +613,7 @@ module Karafka
       rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
         return if breaker.call
 
-        retry if attempt <= app_config.admin.max_attempts
+        retry if attempt <= max_attempts
 
         raise
       end
@@ -613,11 +622,10 @@ module Karafka
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config
       def config(type, settings)
-        app_config
-          .kafka
+        app_kafka
           .then(&:dup)
-          .merge(app_config.admin.kafka)
-          .tap { |config| config[:'group.id'] = app_config.admin.group_id }
+          .merge(admin_kafka)
+          .tap { |config| config[:'group.id'] = group_id }
           # We merge after setting the group id so it can be altered if needed
           # In general in admin we only should alter it when we need to impersonate a given
           # consumer group or do something similar
@@ -651,11 +659,6 @@ module Karafka
           offset
         end
       end
-
-      # @return [Karafka::Core::Configurable::Node] root node config
-      def app_config
-        ::Karafka::App.config
-      end
     end
   end
 end

data/lib/karafka/cli/topics/align.rb

@@ -30,10 +30,13 @@ module Karafka
             return false
           end
 
-          names = resources_to_migrate.map(&:name).join(', ')
-          puts "Updating configuration of the following topics: #{names}"
-          Karafka::Admin::Configs.alter(resources_to_migrate)
-          puts "#{green('Updated')} all requested topics configuration."
+          resources_to_migrate.each do |resource|
+            supervised("Updating topic: #{resource.name} configuration") do
+              Karafka::Admin::Configs.alter(resource)
+            end
+
+            puts "#{green('Updated')} topic #{resource.name} configuration."
+          end
 
           true
         end

data/lib/karafka/cli/topics/base.rb

@@ -12,6 +12,23 @@ module Karafka
 
         private
 
+        # Used to run Karafka Admin commands that talk with Kafka and that can fail due to broker
+        # errors and other issues. We catch errors and provide nicer printed output prior to
+        # re-raising the mapped error for proper exit code status handling
+        #
+        # @param operation_message [String] message that we use to print that it is going to run
+        #   and if case if failed with a failure indication.
+        def supervised(operation_message)
+          puts "#{operation_message}..."
+
+          yield
+        rescue Rdkafka::RdkafkaError => e
+          puts "#{operation_message} #{red('failed')}:"
+          puts e
+
+          raise Errors::CommandValidationError, cause: e
+        end
+
         # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
         # @note If topic is defined in multiple consumer groups, first config will be used. This
         #   means, that this CLI will not work for simultaneous management of multiple clusters

data/lib/karafka/cli/topics/create.rb

@@ -15,13 +15,15 @@ module Karafka
             if existing_topics_names.include?(name)
               puts "#{yellow('Skipping')} because topic #{name} already exists."
             else
-              puts "Creating topic #{name}..."
-              Admin.create_topic(
-                name,
-                topic.declaratives.partitions,
-                topic.declaratives.replication_factor,
-                topic.declaratives.details
-              )
+              supervised("Creating topic #{name}") do
+                Admin.create_topic(
+                  name,
+                  topic.declaratives.partitions,
+                  topic.declaratives.replication_factor,
+                  topic.declaratives.details
+                )
+              end
+
               puts "#{green('Created')} topic #{name}."
               any_created = true
             end

data/lib/karafka/cli/topics/delete.rb

@@ -13,8 +13,10 @@ module Karafka
             name = topic.name
 
             if existing_topics_names.include?(name)
-              puts "Deleting topic #{name}..."
-              Admin.delete_topic(name)
+              supervised("Deleting topic #{name}") do
+                Admin.delete_topic(name)
+              end
+
               puts "#{green('Deleted')} topic #{name}."
               any_deleted = true
             else

data/lib/karafka/cli/topics/help.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Declarative topics CLI sub-help
+      class Help < Base
+        # Displays help information for all available topics management commands
+        def call
+          puts <<~HELP
+            Karafka topics commands:
+              align        # Aligns configuration of all declarative topics based on definitions
+              create       # Creates topics with appropriate settings
+              delete       # Deletes all topics defined in the routes
+              help         # Describes available topics management commands
+              migrate      # Creates missing topics, repartitions existing and aligns configuration
+              plan         # Plans migration process and prints changes to be applied
+              repartition  # Adds additional partitions to topics with fewer partitions than expected
+              reset        # Deletes and re-creates all topics
+
+            Options:
+              --detailed-exitcode  # Provides detailed exit codes (0=no changes, 1=error, 2=changes applied)
+
+            Examples:
+              karafka topics create
+              karafka topics plan --detailed-exitcode
+              karafka topics migrate
+              karafka topics align
+
+            Note: All admin operations run on the default cluster only.
+          HELP
+
+          # We return false to indicate with exit code 0 that no changes were applied
+          false
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/repartition.rb

@@ -21,8 +21,10 @@ module Karafka
             existing_count = existing_partitions.fetch(name, false)
 
             if existing_count && existing_count < desired_count
-              puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
-              Admin.create_partitions(name, desired_count)
+              supervised("Increasing number of partitions to #{desired_count} on topic #{name}") do
+                Admin.create_partitions(name, desired_count)
+              end
+
               change = desired_count - existing_count
               puts "#{green('Created')} #{change} additional partitions on topic #{name}."
               any_repartitioned = true

data/lib/karafka/cli/topics.rb

@@ -27,10 +27,13 @@ module Karafka
       # crashes
       CHANGES_EXIT_CODE = 2
 
-      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE
+      # Used when there was an error during execution.
+      ERROR_EXIT_CODE = 1
+
+      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE, :ERROR_EXIT_CODE
 
       # @param action [String] action we want to take
-      def call(action = 'missing')
+      def call(action = 'help')
         detailed_exit_code = options.fetch(:detailed_exitcode, false)
 
         command = case action
@@ -48,8 +51,10 @@ module Karafka
                     Topics::Align
                   when 'plan'
                     Topics::Plan
+                  when 'help'
+                    Topics::Help
                   else
-                    raise ::ArgumentError, "Invalid topics action: #{action}"
+                    raise Errors::UnrecognizedCommandError, "Unrecognized topics action: #{action}"
                   end
 
         changes = command.new.call
@@ -57,6 +62,8 @@ module Karafka
         return unless detailed_exit_code
 
         changes ? exit(CHANGES_EXIT_CODE) : exit(NO_CHANGES_EXIT_CODE)
+      rescue Errors::CommandValidationError
+        exit(ERROR_EXIT_CODE)
       end
     end
   end

data/lib/karafka/cli.rb

@@ -21,6 +21,8 @@ module Karafka
           args = action ? [action] : []
 
           command.new.call(*args)
+        elsif command_name.nil?
+          Help.new.call
         else
           raise(
             Karafka::Errors::UnrecognizedCommandError,

data/lib/karafka/connection/client.rb

@@ -491,9 +491,17 @@ module Karafka
         #
         # This code adds around 0.01 ms per seek but saves from many user unexpected behaviours in
         # seeking and pausing
-        return if message.offset == topic_partition_position(message.topic, message.partition)
+        position = topic_partition_position(message.topic, message.partition)
 
-        kafka.seek(message)
+        # Always seek if current position cannot be fetched or is negative. Offset seek can also
+        # be negative (-1 or -2) and we should not compare it with the position because they are
+        # special (earliest or latest)
+        return kafka.seek(message) if position.negative?
+        # If offset is the same as the next position, we don't have to seek to get there, hence
+        # only in such case we can do nothing.
+        return kafka.seek(message) if message.offset != position
+
+        nil
       end
 
       # Commits the stored offsets in a sync way and closes the consumer.

data/lib/karafka/constraints.rb

@@ -15,13 +15,13 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.9.0.rc3 because we expect 0.9.0.rc3 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc3')
+        # All good if version higher than 0.10.0 because we expect 0.10.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.10.0')
 
         # If older web-ui used, we cannot allow it
         raise(
           Errors::DependencyConstraintsError,
-          'karafka-web < 0.9.0 is not compatible with this karafka version'
+          'karafka-web < 0.10.0 is not compatible with this karafka version'
         )
       end
 

data/lib/karafka/errors.rb

@@ -22,7 +22,34 @@ module Karafka
     InvalidConfigurationError = Class.new(BaseError)
 
     # Raised when we try to use Karafka CLI commands (except install) without a boot file
-    MissingBootFileError = Class.new(BaseError)
+    MissingBootFileError = Class.new(BaseError) do
+      # @param boot_file_path [Pathname] path where the boot file should be
+      def initialize(boot_file_path)
+        message = <<~MSG
+
+          \e[31mKarafka Boot File Missing:\e[0m #{boot_file_path}
+
+          Cannot find Karafka boot file - this file configures your Karafka application.
+
+          \e[33mQuick fixes:\e[0m
+            \e[32m1.\e[0m Navigate to your Karafka app directory
+            \e[32m2.\e[0m Check if following file exists: \e[36m#{boot_file_path}\e[0m
+            \e[32m3.\e[0m Install Karafka if needed: \e[36mkarafka install\e[0m
+
+          \e[33mCommon causes:\e[0m
+            \e[31m•\e[0m Wrong directory (not in Karafka app root)
+            \e[31m•\e[0m File was accidentally moved or deleted
+            \e[31m•\e[0m New project needing initialization
+
+          For setup help: \e[34mhttps://karafka.io/docs/Getting-Started\e[0m
+        MSG
+
+        super(message)
+        # In case of this error backtrace is irrelevant and we want to print comprehensive error
+        # message without backtrace, this is why nullified.
+        set_backtrace([])
+      end
+    end
 
     # Raised when we've waited enough for shutting down a non-responsive process
     ForcefulShutdownError = Class.new(BaseError)
@@ -65,7 +92,13 @@ module Karafka
     ResultNotVisibleError = Class.new(BaseError)
 
     # Raised when there is an attempt to run an unrecognized CLI command
-    UnrecognizedCommandError = Class.new(BaseError)
+    UnrecognizedCommandError = Class.new(BaseError) do
+      # Overwritten not to print backtrace for unknown CLI command
+      def initialize(*args)
+        super
+        set_backtrace([])
+      end
+    end
 
     # Raised when you were executing a command and it could not finish successfully because of
     # a setup state or parameters configuration

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -8,6 +8,10 @@ module Karafka
     module ScheduledMessages
       # Consumer that coordinates scheduling of messages when the time comes
       class Consumer < ::Karafka::BaseConsumer
+        include Helpers::ConfigImporter.new(
+          dispatcher_class: %i[scheduled_messages dispatcher_class]
+        )
+
         # Prepares the initial state of all stateful components
         def initialized
           clear!
@@ -155,7 +159,7 @@ module Karafka
           @today = Day.new
           @tracker = Tracker.new
           @state = State.new(false)
-          @dispatcher = config.dispatcher_class.new(topic.name, partition)
+          @dispatcher = dispatcher_class.new(topic.name, partition)
           @states_reporter = Helpers::IntervalRunner.new do
             @tracker.today = @daily_buffer.size
             @tracker.state = @state.to_s
@@ -165,11 +169,6 @@ module Karafka
 
           tags.add(:state, @state.to_s)
         end
-
-        # @return [Karafka::Core::Configurable::Node] Schedules config node
-        def config
-          @config ||= Karafka::App.config.scheduled_messages
-        end
       end
     end
   end

data/lib/karafka/server.rb

@@ -9,6 +9,15 @@ module Karafka
 
     private_constant :FORCEFUL_SHUTDOWN_WAIT
 
+    extend Helpers::ConfigImporter.new(
+      cli_contract: %i[internal cli contract],
+      activity_manager: %i[internal routing activity_manager],
+      supervision_sleep: %i[internal supervision_sleep],
+      shutdown_timeout: %i[shutdown_timeout],
+      forceful_exit_code: %i[internal forceful_exit_code],
+      process: %i[internal process]
+    )
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :listeners
@@ -42,9 +51,7 @@ module Karafka
         # embedded
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
-        config.internal.cli.contract.validate!(
-          config.internal.routing.activity_manager.to_h
-        )
+        cli_contract.validate!(activity_manager.to_h)
 
         # We clear as we do not want parent handlers in case of working from fork
         process.clear
@@ -99,18 +106,18 @@ module Karafka
 
         Karafka::App.stop!
 
-        timeout = config.shutdown_timeout
+        timeout = shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
-        ((timeout / 1_000) * (1 / config.internal.supervision_sleep)).to_i.times do
+        ((timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
           all_listeners_stopped = listeners.all?(&:stopped?)
           all_workers_stopped = workers.none?(&:alive?)
 
           return if all_listeners_stopped && all_workers_stopped
 
-          sleep(config.internal.supervision_sleep)
+          sleep(supervision_sleep)
         end
 
         raise Errors::ForcefulShutdownError
@@ -148,7 +155,7 @@ module Karafka
         return unless process.supervised?
 
         # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit!(config.internal.forceful_exit_code)
+        Kernel.exit!(forceful_exit_code)
       ensure
         # We need to check if it wasn't an early exit to make sure that only on stop invocation
         # can change the status after everything is closed
@@ -172,18 +179,6 @@ module Karafka
         # in one direction
         Karafka::App.quiet!
       end
-
-      private
-
-      # @return [Karafka::Core::Configurable::Node] root config node
-      def config
-        Karafka::App.config
-      end
-
-      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
-      def process
-        config.internal.process
-      end
     end
 
     # Always start with standalone so there always is a value for the execution mode.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.0.beta1'
+  VERSION = '2.5.0.beta2'
 end

metadata

@@ -1,39 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.0.beta1
+  version: 2.5.0.beta2
 platform: ruby
 authors:
 - Maciej Mensfeld
 bindir: bin
-cert_chain:
-- |
-  -----BEGIN CERTIFICATE-----
-  MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-  YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
-  dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
-  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
-  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
-  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
-  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
-  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
-  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
-  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
-  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
-  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
-  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
-  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
-  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
-  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
-  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
-  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
-  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
-  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
-  -----END CERTIFICATE-----
+cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -138,6 +111,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
+- ".github/workflows/push.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -152,6 +126,7 @@ files:
 - LICENSE-COMM
 - LICENSE-LGPL
 - README.md
+- Rakefile
 - SECURITY.md
 - bin/benchmarks
 - bin/clean_kafka
@@ -167,7 +142,6 @@ files:
 - bin/verify_license_integrity
 - bin/verify_topics_naming
 - bin/wait_for_kafka
-- certs/cert.pem
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
@@ -207,6 +181,7 @@ files:
 - lib/karafka/cli/topics/base.rb
 - lib/karafka/cli/topics/create.rb
 - lib/karafka/cli/topics/delete.rb
+- lib/karafka/cli/topics/help.rb
 - lib/karafka/cli/topics/migrate.rb
 - lib/karafka/cli/topics/plan.rb
 - lib/karafka/cli/topics/repartition.rb
@@ -644,7 +619,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []

data/certs/cert.pem

@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
-dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
-PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
-ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
-Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
-JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
-T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
-flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
-vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
-iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
-sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
-eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
-Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
-agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
-TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
-Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
-oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
-i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
-ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
------END CERTIFICATE-----

data.tar.gz.sig

@@ -1 +0,0 @@
-`E�����S	�i7�L�(�D��9�A���`�"a)��