karafka 2.2.7 → 2.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6a42d67752052bcf8f29fb683ed8ffb5fce7675c155b73269c4654681913f7b
-  data.tar.gz: b1bfc5bdad87c27111d8aee50302593133460f0eac47161f3c3872dc726d3b68
+  metadata.gz: e440e6a6a067d23095d6c0b60f4cd534080e2fd1cc3e528f1052a6368b60cab0
+  data.tar.gz: d5d1c98a0ac28677b860e74ae0c59b13bba68c07bd84964fb0ca7ac473454fcc
 SHA512:
-  metadata.gz: b1e46475db36dc2fc837aa68b1639ad43b94a59afc72dd7287c60fac5f504f55c1ed68f9a1527a4eb1f6743801d38ba44ac6eef6c9d69a6b2ae4e4fbf3034b98
-  data.tar.gz: 329fe59c9bbede3367c9bb3624c69652111f9387360d28b8d9e83bb0a184ea0e524d87d3f47afc400446f48993a57a41c4c171ae5e16c5413c7a496e88fc0421
+  metadata.gz: f5ea25e23361caba43073fb36c7cf91dacd105ed4a9595929c0e48b608e07701dbcf0be32f8982ae14809a1511f5ddc0cd2ae13fe5263f9ea9d8b2aed538fb16
+  data.tar.gz: 175c25bc43361facb9a7664fbe89fc48a7e7426be8ded047d5a1c549b1db67a0172ee3954972b07adeecf92c3dc4dc2c61e979d9905fa40967133d1872c6a46b

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka framework changelog
 
+## 2.2.8 (2023-10-20)
+- **[Feature]** Introduce Appsignal integration for errors and metrics tracking.
+- [Improvement] Expose `#synchronize` for VPs to allow for locks when cross-VP consumers work is needed.
+- [Improvement] Provide `#collapse_until!` direct consumer API to allow for collapsed virtual partitions consumer operations together with the Filtering API for advanced use-cases.
+- [Refactor] Reorganize how rebalance events are propagated from `librdkafka` to Karafka. Replace `connection.client.rebalance_callback` with `rebalance.partitions_assigned` and `rebalance.partitions_revoked`. Introduce two extra events: `rebalance.partitions_assign` and `rebalance.partitions_revoke` to handle pre-rebalance future work.
+- [Refactor] Remove `thor` as a CLI layer and rely on Ruby `OptParser`
+
+### Upgrade notes
+
+1. Unless you were using `connection.client.rebalance_callback` which was considered private, nothing.
+2. None of the CLI commands should change but `thor` has been removed so please report if you find any bugs.
+
 ## 2.2.7 (2023-10-07)
 - **[Feature]** Introduce Inline Insights to both OSS and Pro. Inline Insights allow you to get the Kafka insights/metrics from the consumer instance and use them to alter the processing flow. In Pro, there's an extra filter flow allowing to ensure, that the insights exist during consumption.
 - [Enhancement] Make sure, that subscription groups ids are unique by including their consumer group id in them similar to how topics ids are handled (not a breaking change).

data/Gemfile.lock

@@ -1,53 +1,63 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.7)
+    karafka (2.2.8)
       karafka-core (>= 2.2.2, < 2.3.0)
-      thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.8)
-      activesupport (= 7.0.8)
+    activejob (7.1.1)
+      activesupport (= 7.1.1)
       globalid (>= 0.3.6)
-    activesupport (7.0.8)
+    activesupport (7.1.1)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
     byebug (11.1.3)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    drb (2.1.1)
+      ruby2_keywords
     erubi (1.12.0)
     factory_bot (6.3.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.5)
+    ffi (1.16.3)
     globalid (1.2.1)
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.2.2)
+    karafka-core (2.2.3)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.5)
+      karafka-rdkafka (>= 0.13.6, < 0.14.0)
+    karafka-rdkafka (0.13.6)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.5)
+    karafka-web (0.7.7)
       erubi (~> 1.4)
-      karafka (>= 2.2.6, < 3.0.0)
+      karafka (>= 2.2.8.beta1, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.4)
     minitest (5.20.0)
+    mutex_m (0.1.2)
     rack (3.0.8)
     rake (13.0.6)
-    roda (3.72.0)
+    roda (3.73.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -62,14 +72,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.1)
+    ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    tilt (2.2.0)
+    tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.6.7)

data/bin/karafka

@@ -6,6 +6,5 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-Karafka::Cli::Base.load
-Karafka::Cli.prepare
-Karafka::Cli.start
+::Karafka::Cli::Base.load
+::Karafka::Cli.start

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.0
+    image: confluentinc/cp-kafka:7.5.1
 
     ports:
       - 9092:9092
@@ -21,3 +21,5 @@ services:
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

data/karafka.gemspec

@@ -22,7 +22,6 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'karafka-core', '>= 2.2.2', '< 2.3.0'
-  spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.6.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
@@ -38,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka',
     'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
     'documentation_uri' => 'https://karafka.io/docs',

data/lib/karafka/base_consumer.rb

@@ -73,6 +73,7 @@ module Karafka
     # @private
     #
     # @return [Boolean] true if there was no exception, otherwise false.
+    #
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.

data/lib/karafka/cli/base.rb

@@ -1,31 +1,17 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # Base class for all the command that we want to define
-    # This base class provides a nicer interface to Thor and allows to easier separate single
-    # independent commands
-    # In order to define a new command you need to:
-    #   - specify its desc
-    #   - implement call method
-    #
-    # @example Create a dummy command
-    #   class Dummy < Base
-    #     self.desc = 'Dummy command'
-    #
-    #     def call
-    #       puts 'I'm doing nothing!
-    #     end
-    #   end
+    # This base class provides an interface to easier separate single independent commands
     class Base
-      include Thor::Shell
+      # @return [Hash] given command cli options
+      attr_reader :options
 
-      # We can use it to call other cli methods via this object
-      attr_reader :cli
-
-      # @param cli [Karafka::Cli] current Karafka Cli instance
-      def initialize(cli)
-        @cli = cli
+      # Creates new CLI command instance
+      def initialize
+        # Parses the given command CLI options
+        @options = self.class.parse_options
       end
 
       # This method should implement proper cli action
@@ -64,26 +50,46 @@ module Karafka
 
         # Allows to set description of a given cli command
         # @param desc [String] Description of a given cli command
-        def desc(desc)
+        def desc(desc = nil)
           @desc ||= desc
         end
 
-        # This method will bind a given Cli command into Karafka Cli
-        # This method is a wrapper to way Thor defines its commands
-        # @param cli_class [Karafka::Cli] Karafka cli_class
-        def bind_to(cli_class)
-          cli_class.desc name, @desc
+        # Allows to set aliases for a given cli command
+        # @param args [Array] list of aliases that we can use to run given cli command
+        def aliases(*args)
+          @aliases ||= []
+          @aliases << args.map(&:to_s)
+        end
+
+        # Parses the CLI options
+        # @return [Hash] hash with parsed values
+        def parse_options
+          options = {}
 
-          (@options || []).each { |option| cli_class.option(*option) }
+          OptionParser.new do |opts|
+            (@options || []).each do |option|
+              # Creates aliases for backwards compatibility
+              names = option[3].flat_map { |name| [name, name.tr('_', '-')] }
+              names.map! { |name| "#{name} value1,value2,valueN" } if option[2] == Array
+              names.uniq!
 
-          context = self
+              opts.on(
+                *[names, option[2], option[1]].flatten
+              ) { |value| options[option[0]] = value }
+            end
+          end.parse!
 
-          cli_class.send :define_method, name do |*args|
-            context.new(self).call(*args)
-          end
+          options
         end
 
-        private
+        # @return [Array<Class>] available commands
+        def commands
+          ObjectSpace
+            .each_object(Class)
+            .select { |klass| klass.superclass == Karafka::Cli::Base }
+            .reject { |klass| klass.to_s.end_with?('::Base') }
+            .sort_by(&:name)
+        end
 
         # @return [String] downcased current class name that we use to define name for
         #   given Cli command
@@ -92,6 +98,11 @@ module Karafka
         def name
           to_s.split('::').last.downcase
         end
+
+        # @return [Array<String>] names and aliases for command matching
+        def names
+          ((@aliases || []) << name).flatten.map(&:to_s)
+        end
       end
     end
   end

data/lib/karafka/cli/console.rb

@@ -2,11 +2,12 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Console Karafka Cli action
     class Console < Base
-      desc 'Start the Karafka console (short-cut alias: "c")'
-      option aliases: 'c'
+      desc 'Starts the Karafka console (short-cut alias: "c")'
+
+      aliases :c
 
       class << self
         # @return [String] Console executing command for non-Rails setup
@@ -25,7 +26,7 @@ module Karafka
 
       # Start the Karafka console
       def call
-        cli.info
+        Info.new.call
 
         command = ::Karafka.rails? ? self.class.rails_console : self.class.console
 

data/lib/karafka/cli/help.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Prints info with list of commands available
+    class Help < Base
+      desc 'Describes available commands'
+
+      # Print available commands
+      def call
+        # Find the longest command for alignment purposes
+        max_command_length = self.class.commands.map(&:name).map(&:size).max
+
+        puts 'Karafka commands:'
+
+        # Print each command formatted with its description
+        self.class.commands.each do |command|
+          puts "  #{command.name.ljust(max_command_length)}    # #{command.desc}"
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -2,10 +2,10 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Info Karafka Cli action
     class Info < Base
-      desc 'Print configuration details and other options of your application'
+      desc 'Prints configuration details and other options of your application'
 
       # Nice karafka banner
       BANNER = <<~BANNER

data/lib/karafka/cli/install.rb

@@ -4,12 +4,12 @@ require 'erb'
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Install Karafka Cli action
     class Install < Base
       include Helpers::Colorize
 
-      desc 'Install all required things for Karafka application in current directory'
+      desc 'Installs all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
@@ -26,9 +26,9 @@ module Karafka
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
-      # @param args [Array] all the things that Thor CLI accepts
-      def initialize(*args)
+      def initialize
         super
+
         dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile

data/lib/karafka/cli/server.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Server Karafka Cli action
     class Server < Base
       include Helpers::Colorize
@@ -12,36 +12,75 @@ module Karafka
 
       private_constant :SUPPORTED_TYPES
 
-      desc 'Start the Karafka server (short-cut alias: "s")'
-
-      option aliases: 's'
-
-      # Thor does not work well with many aliases combinations, hence we remap the aliases
-      # by ourselves in the code
-      option :consumer_groups, type: :array, default: [], aliases: :g
-      option :subscription_groups, type: :array, default: []
-      option :topics, type: :array, default: []
-
-      %i[
-        include
-        exclude
-      ].each do |action|
-        SUPPORTED_TYPES.each do |type|
-          option(
-            "#{action}_#{type}",
-            type: :array,
-            default: []
-          )
-        end
-      end
+      desc 'Starts the Karafka server (short-cut alias: "s")'
+
+      aliases :s
+
+      option(
+        :consumer_groups,
+        'Runs server only with specified consumer groups',
+        Array,
+        %w[
+          -g
+          --consumer_groups
+          --include_consumer_groups
+        ]
+      )
+
+      option(
+        :subscription_groups,
+        'Runs server only with specified subscription groups',
+        Array,
+        %w[
+          --subscription_groups
+          --include_subscription_groups
+        ]
+      )
+
+      option(
+        :topics,
+        'Runs server only with specified topics',
+        Array,
+        %w[
+          --topics
+          --include_topics
+        ]
+      )
+
+      option(
+        :exclude_consumer_groups,
+        'Runs server without specified consumer groups',
+        Array,
+        %w[
+          --exclude_consumer_groups
+        ]
+      )
+
+      option(
+        :exclude_subscription_groups,
+        'Runs server without specified subscription groups',
+        Array,
+        %w[
+          --exclude_subscription_groups
+        ]
+      )
+
+      option(
+        :exclude_topics,
+        'Runs server without specified topics',
+        Array,
+        %w[
+          --exclude_topics
+        ]
+      )
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        register_inclusions(cli)
-        register_exclusions(cli)
+        register_inclusions
+        register_exclusions
 
         Karafka::Server.run
       end
@@ -49,26 +88,22 @@ module Karafka
       private
 
       # Registers things we want to include (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_inclusions(cli)
+      def register_inclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         SUPPORTED_TYPES.each do |type|
-          v1 = cli.options[type] || []
-          v2 = cli.options[:"include_#{type}"] || []
-          names = v1 + v2
+          names = options[type] || []
 
           names.each { |name| activities.include(type, name) }
         end
       end
 
       # Registers things we want to exclude (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_exclusions(cli)
+      def register_exclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         activities.class::SUPPORTED_TYPES.each do |type|
-          names = cli.options[:"exclude_#{type}"] || []
+          names = options[:"exclude_#{type}"] || []
 
           names.each { |name| activities.exclude(type, name) }
         end

data/lib/karafka/cli/topics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # CLI actions related to Kafka cluster topics management
     class Topics < Base
       include Helpers::Colorize

data/lib/karafka/cli.rb

@@ -5,31 +5,35 @@ module Karafka
   #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
-  #
-  # @note Whole Cli is built using Thor
-  # @see https://github.com/erikhuda/thor
-  class Cli < Thor
-    package_name 'Karafka'
-
+  class Cli
     class << self
-      # Loads all Cli commands into Thor framework.
-      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available.
-      def prepare
-        cli_commands.each do |action|
-          action.bind_to(self)
+      # Starts the CLI
+      def start
+        # Command we want to run, like install, server, etc
+        command_name = ARGV[0]
+        # Action for action-based commands like topics migrate
+        action = ARGV[1].to_s.start_with?('-') ? false : ARGV[1]
+
+        command = commands.find { |cmd| cmd.names.include?(command_name) }
+
+        if command
+          # Only actionable commands require command as an argument
+          args = action ? [action] : []
+
+          command.new.call(*args)
+        else
+          raise(
+            Karafka::Errors::UnrecognizedCommandError,
+            "Unrecognized command \"#{command_name}\""
+          )
         end
       end
 
       private
 
-      # @return [Array<Class>] Array with Cli action classes that can be used as commands
-      def cli_commands
-        constants
-          .map! { |object| const_get(object) }
-          .keep_if do |object|
-            object.instance_of?(Class) && (object < Cli::Base)
-          end
+      # @return [Array<Class>] command classes
+      def commands
+        Base.commands
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -43,7 +43,11 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @rebalance_manager = RebalanceManager.new
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(
+          @subscription_group.id,
+          @subscription_group.consumer_group.id
+        )
         @kafka = build_consumer
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
@@ -498,7 +502,8 @@ module Karafka
       def build_consumer
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
-        config.consumer_rebalance_listener = @rebalance_manager
+        config.consumer_rebalance_listener = @rebalance_callback
+
         consumer = config.consumer
         @name = consumer.name
 
@@ -507,7 +512,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Statistics.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )
@@ -517,7 +522,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Error.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )