karafka 2.2.6 → 2.2.8.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36fe50a9b0a758b9b916ca668c862085629bf09bcf16f7c0e8a340a9b7333abb
-  data.tar.gz: 61d3f10df4e1545a1a4e0d8190eb6ee2a7b751636bb347cf2703a74149fc710c
+  metadata.gz: facd28537bbd9b1f5a7e6956c82b704a97efafdfdcea5ed9959991f129ba2ed4
+  data.tar.gz: 4b6ca5058a07fadf7c95fa96eb08ff7820b44d4bafc43ebdaf0a7fe16cb35fd2
 SHA512:
-  metadata.gz: f12fb91f2e8e4e7dba766fbf32c79b734f5cd9f6fa63a66f80863a819dfd6d442024a97a4095bea84ef87bb0cb645ff431ba6d2f04f46d72864c377d61fc76a6
-  data.tar.gz: 4235c1944c98f0bd6359b82b7eab68a69812d70665ab0436c9f67cc5011e4c8edfa2c194a6d65744a4d82f97710beca5ddb871b8671febf61e5c9f591a746bee
+  metadata.gz: fd8cda16faa8d1267b934d2ccc189e6981c21be8cc4fed60042776a6370c93481b99981117d2572a52ada12ea9268877e339a5007284aa2413135ace5a078241
+  data.tar.gz: e765bb3a3c6de52bd6dbd12a7268d18c03cb1454068dff19ebc49d1c5a7850058b8b0853e744c6a16f07bc7f3883bf8681df19a10c0babc70defca1dbdee2b19

data/.github/workflows/ci.yml

@@ -71,6 +71,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           # We run it against the oldest and the newest of a given major to make sure, that there
           # are no syntax-sugars that we would use that were introduced down the road
@@ -115,6 +116,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'
@@ -166,6 +168,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Karafka framework changelog
 
+## 2.2.8 (Unreleased)
+- **[Feature]** Introduce Appsignal integration for errors and metrics tracking.
+- [Improvement] Expose `#synchronize` for VPs to allow for locks when cross-VP consumers work is needed.
+- [Improvement] Provide `#collapse_until!` direct consumer API to allow for collapsed virtual partitions consumer operations together with the Filtering API for advanced use-cases.
+- [Refactor] Reorganize how rebalance events are propagated from `librdkafka` to Karafka. Replace `connection.client.rebalance_callback` with `rebalance.partitions_assigned` and `rebalance.partitions_revoked`. Introduce two extra events: `rebalance.partitions_assign` and `rebalance.partitions_revoke` to handle pre-rebalance future work.
+- [Refactor] Remove `thor` as a CLI layer and rely on Ruby `OptParser`
+
+### Upgrade notes
+
+1. Unless you were using `connection.client.rebalance_callback` which was considered private, nothing.
+2. None of the CLI commands should change but `thor` has been removed so please report if you find any bugs.
+
+## 2.2.7 (2023-10-07)
+- **[Feature]** Introduce Inline Insights to both OSS and Pro. Inline Insights allow you to get the Kafka insights/metrics from the consumer instance and use them to alter the processing flow. In Pro, there's an extra filter flow allowing to ensure, that the insights exist during consumption.
+- [Enhancement] Make sure, that subscription groups ids are unique by including their consumer group id in them similar to how topics ids are handled (not a breaking change).
+- [Enhancement] Expose `#attempt` method on a consumer to directly indicate number of attempt of processing given data.
+- [Enhancement] Support Rails 7.1.
+
 ## 2.2.6 (2023-09-26)
 - [Enhancement] Retry `Karafka::Admin#read_watermark_offsets` fetching upon `not_leader_for_partition` that can occur mostly on newly created topics in KRaft and after crashes during leader selection.
 

data/Gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.6)
+    karafka (2.2.8.beta1)
       karafka-core (>= 2.2.2, < 2.3.0)
-      thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -37,9 +36,9 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.4)
+    karafka-web (0.7.6)
       erubi (~> 1.4)
-      karafka (>= 2.2.3, < 3.0.0)
+      karafka (>= 2.2.6, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -68,14 +67,13 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    tilt (2.2.0)
+    tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.6.7)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-linux

data/bin/karafka

@@ -6,6 +6,5 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-Karafka::Cli::Base.load
-Karafka::Cli.prepare
-Karafka::Cli.start
+::Karafka::Cli::Base.load
+::Karafka::Cli.start

data/config/locales/errors.yml

@@ -16,9 +16,11 @@ en:
       max_wait_time_format: needs to be an integer bigger than 0
       kafka_format: needs to be a filled hash
       internal.processing.jobs_builder_format: cannot be nil
-      internal.processing.scheduler: cannot be nil
-      internal.processing.coordinator_class: cannot be nil
-      internal.processing.partitioner_class: cannot be nil
+      internal.processing.scheduler_format: cannot be nil
+      internal.processing.coordinator_class_format: cannot be nil
+      internal.processing.partitioner_class_format: cannot be nil
+      internal.processing.strategy_selector_format: cannot be nil
+      internal.processing.expansions_selector_format: cannot be nil
       internal.active_job.dispatcher: cannot be nil
       internal.active_job.job_options_contract: cannot be nil
       internal.active_job.consumer_class: cannot be nil
@@ -59,8 +61,9 @@ en:
       initial_offset_format: needs to be either earliest or latest
       subscription_group_name_format: must be a non-empty string
       manual_offset_management.active_format: needs to be either true or false
-      consumer_active_job_missing: ActiveJob needs to be available
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
+      inline_insights.active_format: needs to be either true or false
+      consumer_active_job_missing: ActiveJob needs to be available
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false

data/config/locales/pro_errors.yml

@@ -31,6 +31,9 @@ en:
       patterns.active_format: 'needs to be boolean'
       patterns.type_format: 'needs to be :matcher, :discovered or :regular'
 
+      inline_insights.active_format: 'needs to be boolean'
+      inline_insights.required_format: 'needs to be boolean'
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.0
+    image: confluentinc/cp-kafka:7.5.1
 
     ports:
       - 9092:9092
@@ -21,3 +21,5 @@ services:
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

data/karafka.gemspec

@@ -22,7 +22,6 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'karafka-core', '>= 2.2.2', '< 2.3.0'
-  spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.6.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
@@ -38,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka',
     'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
     'documentation_uri' => 'https://karafka.io/docs',

data/lib/karafka/base_consumer.rb

@@ -73,6 +73,7 @@ module Karafka
     # @private
     #
     # @return [Boolean] true if there was no exception, otherwise false.
+    #
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
@@ -252,7 +253,13 @@ module Karafka
     #   different flow after there is an error, for example for resources cleanup, small manual
     #   backoff or different instrumentation tracking.
     def retrying?
-      coordinator.pause_tracker.attempt > 1
+      attempt > 1
+    end
+
+    # @return [Integer] attempt of processing given batch. 1 if this is the first attempt or higher
+    #  in case it is a retry
+    def attempt
+      coordinator.pause_tracker.attempt
     end
 
     # Pauses the processing from the last offset to retry on given message

data/lib/karafka/cli/base.rb

@@ -1,31 +1,17 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # Base class for all the command that we want to define
-    # This base class provides a nicer interface to Thor and allows to easier separate single
-    # independent commands
-    # In order to define a new command you need to:
-    #   - specify its desc
-    #   - implement call method
-    #
-    # @example Create a dummy command
-    #   class Dummy < Base
-    #     self.desc = 'Dummy command'
-    #
-    #     def call
-    #       puts 'I'm doing nothing!
-    #     end
-    #   end
+    # This base class provides an interface to easier separate single independent commands
     class Base
-      include Thor::Shell
+      # @return [Hash] given command cli options
+      attr_reader :options
 
-      # We can use it to call other cli methods via this object
-      attr_reader :cli
-
-      # @param cli [Karafka::Cli] current Karafka Cli instance
-      def initialize(cli)
-        @cli = cli
+      # Creates new CLI command instance
+      def initialize
+        # Parses the given command CLI options
+        @options = self.class.parse_options
       end
 
       # This method should implement proper cli action
@@ -64,26 +50,46 @@ module Karafka
 
         # Allows to set description of a given cli command
         # @param desc [String] Description of a given cli command
-        def desc(desc)
+        def desc(desc = nil)
           @desc ||= desc
         end
 
-        # This method will bind a given Cli command into Karafka Cli
-        # This method is a wrapper to way Thor defines its commands
-        # @param cli_class [Karafka::Cli] Karafka cli_class
-        def bind_to(cli_class)
-          cli_class.desc name, @desc
+        # Allows to set aliases for a given cli command
+        # @param args [Array] list of aliases that we can use to run given cli command
+        def aliases(*args)
+          @aliases ||= []
+          @aliases << args.map(&:to_s)
+        end
+
+        # Parses the CLI options
+        # @return [Hash] hash with parsed values
+        def parse_options
+          options = {}
 
-          (@options || []).each { |option| cli_class.option(*option) }
+          OptionParser.new do |opts|
+            (@options || []).each do |option|
+              # Creates aliases for backwards compatibility
+              names = option[3].flat_map { |name| [name, name.tr('_', '-')] }
+              names.map! { |name| "#{name} value1,value2,valueN" } if option[2] == Array
+              names.uniq!
 
-          context = self
+              opts.on(
+                *[names, option[2], option[1]].flatten
+              ) { |value| options[option[0]] = value }
+            end
+          end.parse!
 
-          cli_class.send :define_method, name do |*args|
-            context.new(self).call(*args)
-          end
+          options
         end
 
-        private
+        # @return [Array<Class>] available commands
+        def commands
+          ObjectSpace
+            .each_object(Class)
+            .select { |klass| klass.superclass == Karafka::Cli::Base }
+            .reject { |klass| klass.to_s.end_with?('::Base') }
+            .sort_by(&:name)
+        end
 
         # @return [String] downcased current class name that we use to define name for
         #   given Cli command
@@ -92,6 +98,11 @@ module Karafka
         def name
           to_s.split('::').last.downcase
         end
+
+        # @return [Array<String>] names and aliases for command matching
+        def names
+          ((@aliases || []) << name).flatten.map(&:to_s)
+        end
       end
     end
   end

data/lib/karafka/cli/console.rb

@@ -2,11 +2,12 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Console Karafka Cli action
     class Console < Base
-      desc 'Start the Karafka console (short-cut alias: "c")'
-      option aliases: 'c'
+      desc 'Starts the Karafka console (short-cut alias: "c")'
+
+      aliases :c
 
       class << self
         # @return [String] Console executing command for non-Rails setup
@@ -25,7 +26,7 @@ module Karafka
 
       # Start the Karafka console
       def call
-        cli.info
+        Info.new.call
 
         command = ::Karafka.rails? ? self.class.rails_console : self.class.console
 

data/lib/karafka/cli/help.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Prints info with list of commands available
+    class Help < Base
+      desc 'Describes available commands'
+
+      # Print available commands
+      def call
+        # Find the longest command for alignment purposes
+        max_command_length = self.class.commands.map(&:name).map(&:size).max
+
+        puts 'Karafka commands:'
+
+        # Print each command formatted with its description
+        self.class.commands.each do |command|
+          puts "  #{command.name.ljust(max_command_length)}    # #{command.desc}"
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -2,10 +2,10 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Info Karafka Cli action
     class Info < Base
-      desc 'Print configuration details and other options of your application'
+      desc 'Prints configuration details and other options of your application'
 
       # Nice karafka banner
       BANNER = <<~BANNER

data/lib/karafka/cli/install.rb

@@ -4,12 +4,12 @@ require 'erb'
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Install Karafka Cli action
     class Install < Base
       include Helpers::Colorize
 
-      desc 'Install all required things for Karafka application in current directory'
+      desc 'Installs all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
@@ -26,9 +26,9 @@ module Karafka
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
-      # @param args [Array] all the things that Thor CLI accepts
-      def initialize(*args)
+      def initialize
         super
+
         dependencies = Bundler::LockfileParser.new(
           Bundler.read_file(
             Bundler.default_lockfile

data/lib/karafka/cli/server.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Server Karafka Cli action
     class Server < Base
       include Helpers::Colorize
@@ -12,36 +12,75 @@ module Karafka
 
       private_constant :SUPPORTED_TYPES
 
-      desc 'Start the Karafka server (short-cut alias: "s")'
-
-      option aliases: 's'
-
-      # Thor does not work well with many aliases combinations, hence we remap the aliases
-      # by ourselves in the code
-      option :consumer_groups, type: :array, default: [], aliases: :g
-      option :subscription_groups, type: :array, default: []
-      option :topics, type: :array, default: []
-
-      %i[
-        include
-        exclude
-      ].each do |action|
-        SUPPORTED_TYPES.each do |type|
-          option(
-            "#{action}_#{type}",
-            type: :array,
-            default: []
-          )
-        end
-      end
+      desc 'Starts the Karafka server (short-cut alias: "s")'
+
+      aliases :s
+
+      option(
+        :consumer_groups,
+        'Runs server only with specified consumer groups',
+        Array,
+        %w[
+          -g
+          --consumer_groups
+          --include_consumer_groups
+        ]
+      )
+
+      option(
+        :subscription_groups,
+        'Runs server only with specified subscription groups',
+        Array,
+        %w[
+          --subscription_groups
+          --include_subscription_groups
+        ]
+      )
+
+      option(
+        :topics,
+        'Runs server only with specified topics',
+        Array,
+        %w[
+          --topics
+          --include_topics
+        ]
+      )
+
+      option(
+        :exclude_consumer_groups,
+        'Runs server without specified consumer groups',
+        Array,
+        %w[
+          --exclude_consumer_groups
+        ]
+      )
+
+      option(
+        :exclude_subscription_groups,
+        'Runs server without specified subscription groups',
+        Array,
+        %w[
+          --exclude_subscription_groups
+        ]
+      )
+
+      option(
+        :exclude_topics,
+        'Runs server without specified topics',
+        Array,
+        %w[
+          --exclude_topics
+        ]
+      )
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        register_inclusions(cli)
-        register_exclusions(cli)
+        register_inclusions
+        register_exclusions
 
         Karafka::Server.run
       end
@@ -49,26 +88,22 @@ module Karafka
       private
 
       # Registers things we want to include (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_inclusions(cli)
+      def register_inclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         SUPPORTED_TYPES.each do |type|
-          v1 = cli.options[type] || []
-          v2 = cli.options[:"include_#{type}"] || []
-          names = v1 + v2
+          names = options[type] || []
 
           names.each { |name| activities.include(type, name) }
         end
       end
 
       # Registers things we want to exclude (if defined)
-      # @param cli [Karafka::Cli] Thor cli handler
-      def register_exclusions(cli)
+      def register_exclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
 
         activities.class::SUPPORTED_TYPES.each do |type|
-          names = cli.options[:"exclude_#{type}"] || []
+          names = options[:"exclude_#{type}"] || []
 
           names.each { |name| activities.exclude(type, name) }
         end

data/lib/karafka/cli/topics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  class Cli < Thor
+  class Cli
     # CLI actions related to Kafka cluster topics management
     class Topics < Base
       include Helpers::Colorize

data/lib/karafka/cli.rb

@@ -5,31 +5,35 @@ module Karafka
   #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
-  #
-  # @note Whole Cli is built using Thor
-  # @see https://github.com/erikhuda/thor
-  class Cli < Thor
-    package_name 'Karafka'
-
+  class Cli
     class << self
-      # Loads all Cli commands into Thor framework.
-      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available.
-      def prepare
-        cli_commands.each do |action|
-          action.bind_to(self)
+      # Starts the CLI
+      def start
+        # Command we want to run, like install, server, etc
+        command_name = ARGV[0]
+        # Action for action-based commands like topics migrate
+        action = ARGV[1].to_s.start_with?('-') ? false : ARGV[1]
+
+        command = commands.find { |cmd| cmd.names.include?(command_name) }
+
+        if command
+          # Only actionable commands require command as an argument
+          args = action ? [action] : []
+
+          command.new.call(*args)
+        else
+          raise(
+            Karafka::Errors::UnrecognizedCommandError,
+            "Unrecognized command \"#{command_name}\""
+          )
         end
       end
 
       private
 
-      # @return [Array<Class>] Array with Cli action classes that can be used as commands
-      def cli_commands
-        constants
-          .map! { |object| const_get(object) }
-          .keep_if do |object|
-            object.instance_of?(Class) && (object < Cli::Base)
-          end
+      # @return [Array<Class>] command classes
+      def commands
+        Base.commands
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -43,7 +43,11 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @rebalance_manager = RebalanceManager.new
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(
+          @subscription_group.id,
+          @subscription_group.consumer_group.id
+        )
         @kafka = build_consumer
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
@@ -498,7 +502,8 @@ module Karafka
       def build_consumer
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
-        config.consumer_rebalance_listener = @rebalance_manager
+        config.consumer_rebalance_listener = @rebalance_callback
+
         consumer = config.consumer
         @name = consumer.name
 
@@ -507,7 +512,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Statistics.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )
@@ -517,7 +522,7 @@ module Karafka
           @subscription_group.id,
           Instrumentation::Callbacks::Error.new(
             @subscription_group.id,
-            @subscription_group.consumer_group_id,
+            @subscription_group.consumer_group.id,
             @name
           )
         )