karafka 2.2.7 → 2.2.8.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6a42d67752052bcf8f29fb683ed8ffb5fce7675c155b73269c4654681913f7b
-  data.tar.gz: b1bfc5bdad87c27111d8aee50302593133460f0eac47161f3c3872dc726d3b68
+  metadata.gz: facd28537bbd9b1f5a7e6956c82b704a97efafdfdcea5ed9959991f129ba2ed4
+  data.tar.gz: 4b6ca5058a07fadf7c95fa96eb08ff7820b44d4bafc43ebdaf0a7fe16cb35fd2
 SHA512:
-  metadata.gz: b1e46475db36dc2fc837aa68b1639ad43b94a59afc72dd7287c60fac5f504f55c1ed68f9a1527a4eb1f6743801d38ba44ac6eef6c9d69a6b2ae4e4fbf3034b98
-  data.tar.gz: 329fe59c9bbede3367c9bb3624c69652111f9387360d28b8d9e83bb0a184ea0e524d87d3f47afc400446f48993a57a41c4c171ae5e16c5413c7a496e88fc0421
+  metadata.gz: fd8cda16faa8d1267b934d2ccc189e6981c21be8cc4fed60042776a6370c93481b99981117d2572a52ada12ea9268877e339a5007284aa2413135ace5a078241
+  data.tar.gz: e765bb3a3c6de52bd6dbd12a7268d18c03cb1454068dff19ebc49d1c5a7850058b8b0853e744c6a16f07bc7f3883bf8681df19a10c0babc70defca1dbdee2b19

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka framework changelog
 
+## 2.2.8 (Unreleased)
+- **[Feature]** Introduce Appsignal integration for errors and metrics tracking.
+- [Improvement] Expose `#synchronize` for VPs to allow for locks when cross-VP consumers work is needed.
+- [Improvement] Provide `#collapse_until!` direct consumer API to allow for collapsed virtual partitions consumer operations together with the Filtering API for advanced use-cases.
+- [Refactor] Reorganize how rebalance events are propagated from `librdkafka` to Karafka. Replace `connection.client.rebalance_callback` with `rebalance.partitions_assigned` and `rebalance.partitions_revoked`. Introduce two extra events: `rebalance.partitions_assign` and `rebalance.partitions_revoke` to handle pre-rebalance future work.
+- [Refactor] Remove `thor` as a CLI layer and rely on Ruby `OptParser`
+
+### Upgrade notes
+
+1. Unless you were using `connection.client.rebalance_callback` which was considered private, nothing.
+2. None of the CLI commands should change but `thor` has been removed so please report if you find any bugs.
+
 ## 2.2.7 (2023-10-07)
 - **[Feature]** Introduce Inline Insights to both OSS and Pro. Inline Insights allow you to get the Kafka insights/metrics from the consumer instance and use them to alter the processing flow. In Pro, there's an extra filter flow allowing to ensure, that the insights exist during consumption.
 - [Enhancement] Make sure, that subscription groups ids are unique by including their consumer group id in them similar to how topics ids are handled (not a breaking change).

data/Gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.7)
+    karafka (2.2.8.beta1)
       karafka-core (>= 2.2.2, < 2.3.0)
-      thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -37,7 +36,7 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.5)
+    karafka-web (0.7.6)
       erubi (~> 1.4)
       karafka (>= 2.2.6, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
@@ -68,8 +67,7 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    tilt (2.2.0)
+    tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.6.7)

data/bin/karafka

@@ -6,6 +6,5 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-Karafka::Cli::Base.load
-Karafka::Cli.prepare
-Karafka::Cli.start
+::Karafka::Cli::Base.load
+::Karafka::Cli.start

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.0
+    image: confluentinc/cp-kafka:7.5.1
 
     ports:
       - 9092:9092
@@ -21,3 +21,5 @@ services:
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

data/karafka.gemspec

@@ -22,7 +22,6 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'karafka-core', '>= 2.2.2', '< 2.3.0'
-  spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.6.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
@@ -38,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka',
     'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
     'documentation_uri' => 'https://karafka.io/docs',

data/lib/karafka/base_consumer.rb

@@ -73,6 +73,7 @@ module Karafka
     # @private
     #
     # @return [Boolean] true if there was no exception, otherwise false.
+    #
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.

data/lib/karafka/cli/base.rb

@@ -1,31 +1,17 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # Base class for all the command that we want to define
-    # This base class provides a nicer interface to Thor and allows to easier separate single
-    # independent commands
-    # In order to define a new command you need to:
-    #   - specify its desc
-    #   - implement call method
-    #
-    # @example Create a dummy command
-    #   class Dummy < Base
-    #     self.desc = 'Dummy command'
-    #
-    #     def call
-    #       puts 'I'm doing nothing!
-    #     end
-    #   end
+    # This base class provides an interface to easier separate single independent commands
     class Base
-      include Thor::Shell
+      # @return [Hash] given command cli options
+      attr_reader :options
 
-      # We can use it to call other cli methods via this object
-      attr_reader :cli
-
-      # @param cli [Karafka::Cli] current Karafka Cli instance
-      def initialize(cli)
-        @cli = cli
+      # Creates new CLI command instance
+      def initialize
+        # Parses the given command CLI options
+        @options = self.class.parse_options
       end
 
       # This method should implement proper cli action
@@ -64,26 +50,46 @@ module Karafka
 
         # Allows to set description of a given cli command
         # @param desc [String] Description of a given cli command
-        def desc(desc)
+        def desc(desc = nil)
           @desc ||= desc
         end
 
-        # This method will bind a given Cli command into Karafka Cli
-        # This method is a wrapper to way Thor defines its commands
-        # @param cli_class [Karafka::Cli] Karafka cli_class
-        def bind_to(cli_class)
-          cli_class.desc name, @desc
+        # Allows to set aliases for a given cli command
+        # @param args [Array] list of aliases that we can use to run given cli command
+        def aliases(*args)
+          @aliases ||= []
+          @aliases << args.map(&:to_s)
+        end
+
+        # Parses the CLI options
+        # @return [Hash] hash with parsed values
+        def parse_options
+          options = {}
 
-          (@options || []).each { |option| cli_class.option(*option) }
+          OptionParser.new do |opts|
+            (@options || []).each do |option|
+              # Creates aliases for backwards compatibility
+              names = option[3].flat_map { |name| [name, name.tr('_', '-')] }
+              names.map! { |name| "#{name} value1,value2,valueN" } if option[2] == Array
+              names.uniq!
 
-          context = self
+              opts.on(
+                *[names, option[2], option[1]].flatten
+              ) { |value| options[option[0]] = value }
+            end
+          end.parse!
 
-          cli_class.send :define_method, name do |*args|
-            context.new(self).call(*args)
-          end
+          options
         end
 
-        private
+        # @return [Array<Class>] available commands
+        def commands
+          ObjectSpace
+            .each_object(Class)
+            .select { |klass| klass.superclass == Karafka::Cli::Base }
+            .reject { |klass| klass.to_s.end_with?('::Base') }
+            .sort_by(&:name)
+        end
 
         # @return [String] downcased current class name that we use to define name for
         #   given Cli command
@@ -92,6 +98,11 @@ module Karafka
         def name
           to_s.split('::').last.downcase
         end
+
+        # @return [Array<String>] names and aliases for command matching
+        def names
+          ((@aliases || []) << name).flatten.map(&:to_s)
+        end
       end
     end
   end

data/lib/karafka/cli/console.rb

@@ -2,11 +2,12 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Console Karafka Cli action
     class Console < Base
-      desc 'Start the Karafka console (short-cut alias: "c")'
-      option aliases: 'c'
+      desc 'Starts the Karafka console (short-cut alias: "c")'
+
+      aliases :c
 
       class << self
         # @return [String] Console executing command for non-Rails setup
@@ -25,7 +26,7 @@ module Karafka
 
       # Start the Karafka console
       def call
-        cli.info
+        Info.new.call
 
         command = ::Karafka.rails? ? self.class.rails_console : self.class.console
 

data/lib/karafka/cli/help.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Prints info with list of commands available
+    class Help < Base
+      desc 'Describes available commands'
+
+      # Print available commands
+      def call
+        # Find the longest command for alignment purposes
+        max_command_length = self.class.commands.map(&:name).map(&:size).max
+
+        puts 'Karafka commands:'
+
+        # Print each command formatted with its description
+        self.class.commands.each do |command|
+          puts "  #{command.name.ljust(max_command_length)}    # #{command.desc}"
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -2,10 +2,10 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Info Karafka Cli action
     class Info < Base
-      desc 'Print configuration details and other options of your application'
+      desc 'Prints configuration details and other options of your application'
 
       # Nice karafka banner
       BANNER = <<~BANNER

data/lib/karafka/cli/install.rb

@@ -4,12 +4,12 @@ require 'erb'
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Install Karafka Cli action
     class Install < Base
       include Helpers::Colorize
 
-      desc 'Install all required things for Karafka application in current directory'
+      desc 'Installs all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
@@ -26,9 +26,9 @@ module Karafka
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
-      # @param args [Array] all the things that Thor CLI accepts
-      def initialize(*args)
+      def initialize
         super
+
         dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile

data/lib/karafka/cli/server.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Server Karafka Cli action
     class Server < Base
       include Helpers::Colorize
@@ -12,36 +12,75 @@ module Karafka
 
       private_constant :SUPPORTED_TYPES
 
-      desc 'Start the Karafka server (short-cut alias: "s")'
-
-      option aliases: 's'
-
-      # Thor does not work well with many aliases combinations, hence we remap the aliases
-      # by ourselves in the code
-      option :consumer_groups, type: :array, default: [], aliases: :g
-      option :subscription_groups, type: :array, default: []
-      option :topics, type: :array, default: []
-
-      %i[
-        include
-        exclude
-      ].each do |action|
-        SUPPORTED_TYPES.each do |type|
-          option(
-            "#{action}_#{type}",
-            type: :array,
-            default: []
-          )
-        end
-      end
+      desc 'Starts the Karafka server (short-cut alias: "s")'
+
+      aliases :s
+
+      option(
+        :consumer_groups,
+        'Runs server only with specified consumer groups',
+        Array,
+        %w[
+          -g
+          --consumer_groups
+          --include_consumer_groups
+        ]
+      )
+
+      option(
+        :subscription_groups,
+        'Runs server only with specified subscription groups',
+        Array,
+        %w[
+          --subscription_groups
+          --include_subscription_groups
+        ]
+      )
+
+      option(
+        :topics,
+        'Runs server only with specified topics',
+        Array,
+        %w[
+          --topics
+          --include_topics
+        ]
+      )
+
+      option(
+        :exclude_consumer_groups,
+        'Runs server without specified consumer groups',
+        Array,
+        %w[
+          --exclude_consumer_groups
+        ]
+      )
+
+      option(
+        :exclude_subscription_groups,
+        'Runs server without specified subscription groups',
+        Array,
+        %w[
+          --exclude_subscription_groups
+        ]
+      )
+
+      option(
+        :exclude_topics,
+        'Runs server without specified topics',
+        Array,
+        %w[
+          --exclude_topics
+        ]
+      )
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        register_inclusions(cli)
-        register_exclusions(cli)
+        register_inclusions
+        register_exclusions
 
         Karafka::Server.run
       end
@@ -49,26 +88,22 @@ module Karafka
       private
 
       # Registers things we want to include (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_inclusions(cli)
+      def register_inclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         SUPPORTED_TYPES.each do |type|
-          v1 = cli.options[type] || []
-          v2 = cli.options[:"include_#{type}"] || []
-          names = v1 + v2
+          names = options[type] || []
 
           names.each { |name| activities.include(type, name) }
         end
       end
 
       # Registers things we want to exclude (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_exclusions(cli)
+      def register_exclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         activities.class::SUPPORTED_TYPES.each do |type|
-          names = cli.options[:"exclude_#{type}"] || []
+          names = options[:"exclude_#{type}"] || []
 
           names.each { |name| activities.exclude(type, name) }
         end

data/lib/karafka/cli/topics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # CLI actions related to Kafka cluster topics management
     class Topics < Base
       include Helpers::Colorize

data/lib/karafka/cli.rb

@@ -5,31 +5,35 @@ module Karafka
   #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
-  #
-  # @note Whole Cli is built using Thor
-  # @see https://github.com/erikhuda/thor
-  class Cli < Thor
-    package_name 'Karafka'
-
+  class Cli
     class << self
-      # Loads all Cli commands into Thor framework.
-      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available.
-      def prepare
-        cli_commands.each do |action|
-          action.bind_to(self)
+      # Starts the CLI
+      def start
+        # Command we want to run, like install, server, etc
+        command_name = ARGV[0]
+        # Action for action-based commands like topics migrate
+        action = ARGV[1].to_s.start_with?('-') ? false : ARGV[1]
+
+        command = commands.find { |cmd| cmd.names.include?(command_name) }
+
+        if command
+          # Only actionable commands require command as an argument
+          args = action ? [action] : []
+
+          command.new.call(*args)
+        else
+          raise(
+            Karafka::Errors::UnrecognizedCommandError,
+            "Unrecognized command \"#{command_name}\""
+          )
         end
       end
 
       private
 
-      # @return [Array<Class>] Array with Cli action classes that can be used as commands
-      def cli_commands
-        constants
-          .map! { |object| const_get(object) }
-          .keep_if do |object|
-            object.instance_of?(Class) && (object < Cli::Base)
-          end
+      # @return [Array<Class>] command classes
+      def commands
+        Base.commands
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -43,7 +43,11 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @rebalance_manager = RebalanceManager.new
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(
+          @subscription_group.id,
+          @subscription_group.consumer_group.id
+        )
         @kafka = build_consumer
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
@@ -498,7 +502,8 @@ module Karafka
       def build_consumer
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
-        config.consumer_rebalance_listener = @rebalance_manager
+        config.consumer_rebalance_listener = @rebalance_callback
+
         consumer = config.consumer
         @name = consumer.name
 
@@ -507,7 +512,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Statistics.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )
@@ -517,7 +522,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Error.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )

data/lib/karafka/connection/rebalance_manager.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Connection
-    # Manager for tracking changes in the partitions assignment.
+    # Manager for tracking changes in the partitions assignment after the assignment is done.
     #
     # We need tracking of those to clean up consumers that will no longer process given partitions
     # as they were taken away.
@@ -17,6 +17,10 @@ module Karafka
     #
     # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
     #   partitions, not all the partitions that are owned
+    #
+    # @note We have to have the `subscription_group` reference because we have a global pipeline
+    #   for notifications and we need to make sure we track changes only for things that are of
+    #   relevance to our subscription group
     class RebalanceManager
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze
@@ -25,12 +29,17 @@ module Karafka
 
       private_constant :EMPTY_ARRAY
 
+      # @param subscription_group_id [String] subscription group id
       # @return [RebalanceManager]
-      def initialize
+      def initialize(subscription_group_id)
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
         @active = false
+        @subscription_group_id = subscription_group_id
+
+        # Connects itself to the instrumentation pipeline so rebalances can be tracked
+        ::Karafka.monitor.subscribe(self)
       end
 
       # Resets the rebalance manager state
@@ -55,36 +64,42 @@ module Karafka
         @active
       end
 
-      # Callback that kicks in inside of rdkafka, when new partitions are assigned.
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
+      end
+
+      # Callback that kicks in inside of rdkafka, when new partitions were assigned.
       #
       # @private
-      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_assigned(partitions)
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_assigned(event)
+        # Apply changes only for our subscription group
+        return unless event[:subscription_group_id] == @subscription_group_id
+
         @active = true
-        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
 
-      # Callback that kicks in inside of rdkafka, when partitions are revoked.
+      # Callback that kicks in inside of rdkafka, when partitions were revoked.
       #
       # @private
-      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_revoked(partitions)
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_revoked(event)
+        # Apply changes only for our subscription group
+        return unless event[:subscription_group_id] == @subscription_group_id
+
         @active = true
-        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
-
-      # We consider as lost only partitions that were taken away and not re-assigned back to us
-      def lost_partitions
-        lost_partitions = {}
-
-        revoked_partitions.each do |topic, partitions|
-          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        lost_partitions
-      end
     end
   end
 end