karafka 2.4.2 → 2.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5eb2aeb0bd3cd0d50d70b0fd792e8290315f3131222d3befab9b5a05bd478e07
-  data.tar.gz: 27b4d40fe18c3daf87ea3f2b601487d6cad5a766121374772179fc5fb3049f75
+  metadata.gz: c37b855609a9d20ca0b5cf13875d30ccb71b4072b3f07886f10445ce761c6e1e
+  data.tar.gz: 852fa3340f0dd1cbc2286823a2451c635568440c3e753e7235a72dc85289fcf5
 SHA512:
-  metadata.gz: af5c4030ccd1f881558df342c3936152efc6183ccc6238246a327d6da8b716e65e3e56ec198a708b87f292bd01d9e426fb64b3c4b9a4f5b40111ee68795ec050
-  data.tar.gz: 7d04bb79ea03cc5eedf7a8e9ecd62abc0ff2b48aea2e4df929d691b931b728017adf843aec5c248a6d4e8b42dd1957c798e799b9df82ae009795ae6b414532ac
+  metadata.gz: 373b460007cefdb039b5f2fd00abdc722f42c5bd8f324dad121c7e293264ca4a63cb63cb4fa5057e0f2ff856fe6c28073c04d2ffb21250dd534408ee511807c9
+  data.tar.gz: 595e408578801d949bb2b1e8d3337e8535473b80c7a1db34e56822214de9fd2083a594433e264ed8675c2292cd3aa73bacaef44938ad7a4cd743292097fd4d8f

data/.github/workflows/ci.yml

@@ -73,6 +73,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -113,6 +114,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -166,6 +168,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.3

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka framework changelog
 
+## 2.4.3 (2024-06-12)
+- [Enhancement] Allow for customization of Virtual Partitions reducer for enhanced parallelization.
+- [Enhancement] Add more error codes to early report on polling issues (kidlab)
+- [Enhancement] Add `transport`, `network_exception` and `coordinator_load_in_progress` alongside `timed_out` to retryable errors for the proxy.
+- [Enhancement] Improve `strict_topics_namespacing` validation message.
+- [Change] Remove default empty thread name from `Async` since Web has been upgraded.
+- [Fix] Installer doesn't respect directories in `KARAFKA_BOOT_FILE`.
+- [Fix] Fix case where non absolute boot file path would not work as expected.
+- [Fix] Allow for installing Karafka in a non-existing (yet) directory
+- [Maintenance] Require `waterdrop` `>=` `2.7.3` to support idempotent producer detection.
+
 ## 2.4.2 (2024-05-14)
 - [Enhancement] Validate ActiveJob adapter custom producer format.
 - [Fix] Internal seek does not resolve the offset correctly for time based lookup.

data/Gemfile

@@ -19,6 +19,7 @@ end
 group :test do
   gem 'byebug'
   gem 'factory_bot'
+  gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.2)
+    karafka (2.4.3)
       base64 (~> 0.2)
       karafka-core (>= 2.4.0, < 2.5.0)
-      waterdrop (>= 2.7.0, < 3.0.0)
+      waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.3.2)
-      activesupport (= 7.1.3.2)
+    activejob (7.1.3.4)
+      activesupport (= 7.1.3.4)
       globalid (>= 0.3.6)
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.4)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -26,7 +26,7 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.1)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
@@ -34,7 +34,7 @@ GEM
     erubi (1.12.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
-    ffi (1.16.3)
+    ffi (1.17.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
     i18n (1.14.5)
@@ -54,12 +54,13 @@ GEM
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    mini_portile2 (2.8.6)
-    minitest (5.22.3)
+    mini_portile2 (2.8.7)
+    minitest (5.23.1)
     mutex_m (0.2.0)
+    ostruct (0.6.0)
     rack (3.0.11)
     rake (13.2.1)
-    roda (3.79.0)
+    roda (3.80.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -83,11 +84,11 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.2)
+    waterdrop (2.7.3)
       karafka-core (>= 2.4.0, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.15)
 
 PLATFORMS
   ruby
@@ -100,8 +101,9 @@ DEPENDENCIES
   karafka!
   karafka-testing (>= 2.4.0)
   karafka-web (>= 0.9.0)
+  ostruct
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.9
+   2.5.11

data/config/locales/errors.yml

@@ -121,8 +121,9 @@ en:
       declaratives.details_format: needs to be a hash with only symbol keys
 
       inconsistent_namespacing: |
-        needs to be consistent namespacing style
-        disable this validation by setting config.strict_topics_namespacing to false
+        needs to follow a consistent namespacing style using either dots (.) or underscores (_), but not both.
+        This ensures proper Kafka metrics reporting and avoids name collisions.
+        To disable this validation, set config.strict_topics_namespacing to false.
 
       deserializers.active_format: 'needs to be true'
       deserializers.payload_format: 'needs to respond to #call'

data/config/locales/pro_errors.yml

@@ -4,6 +4,7 @@ en:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
+      virtual_partitions.reducer_format: "needs to respond to `#call`"
 
       long_running_job.active_format: needs to be either true or false
 

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
-  spec.add_dependency 'waterdrop', '>= 2.7.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

data/lib/karafka/cli/install.rb

@@ -21,7 +21,7 @@ module Karafka
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'karafka.rb.erb' => Karafka.boot_file,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
@@ -51,6 +51,7 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           pathed_target = Karafka.root.join(target)
+          FileUtils.mkdir_p File.dirname(pathed_target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
           render = ::ERB.new(template, trim_mode: '-').result(binding)

data/lib/karafka/connection/client.rb

@@ -553,6 +553,12 @@ module Karafka
           early_report = true
         when :transport # -195
           early_report = true
+        when :topic_authorization_failed # 29
+          early_report = true
+        when :group_authorization_failed # 30
+          early_report = true
+        when :cluster_authorization_failed # 31
+          early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
         # This will be raised each time poll detects a non-existing topic. When auto creation is

data/lib/karafka/connection/proxy.rb

@@ -16,8 +16,11 @@ module Karafka
       RETRYABLE_DEFAULT_ERRORS = %i[
         all_brokers_down
         timed_out
+        transport
+        network_exception
         not_coordinator
         not_leader_for_partition
+        coordinator_load_in_progress
       ].freeze
 
       private_constant :RETRYABLE_DEFAULT_ERRORS

data/lib/karafka/helpers/async.rb

@@ -28,7 +28,7 @@ module Karafka
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name = '')
+      def async_call(thread_name)
         MUTEX.synchronize do
           return if @thread&.alive?
 

data/lib/karafka/pro/processing/partitioner.rb

@@ -25,6 +25,8 @@ module Karafka
         def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
+          vps = ktopic.virtual_partitions
+
           # We only partition work if we have:
           # - a virtual partitioner
           # - more than one thread to process the data
@@ -38,21 +40,19 @@ module Karafka
           #
           # This is great because it allows us to run things without the parallelization that adds
           # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
-          if ktopic.virtual_partitions? &&
-             ktopic.virtual_partitions.max_partitions > 1 &&
-             !coordinator.collapsed?
-            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
-            # of the end user action. Otherwise the persistence layer for consumers would cache
-            # it forever and it would cause memory leaks
-            #
-            # This also needs to be consistent because the aggregation here needs to warrant, that
-            # the same partitioned message will always be assigned to the same virtual partition.
-            # Otherwise in case of a window aggregation with VP spanning across several polls, the
-            # data could not be complete.
+          if vps.active? && vps.max_partitions > 1 && !coordinator.collapsed?
             groupings = messages.group_by do |msg|
-              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
-
-              key % ktopic.virtual_partitions.max_partitions
+              # We need to reduce it to the max concurrency, so the group_id is not a direct effect
+              # of the end user action. Otherwise the persistence layer for consumers would cache
+              # it forever and it would cause memory leaks
+              #
+              # This also needs to be consistent because the aggregation here needs to warrant,
+              # that the same partitioned message will always be assigned to the same virtual
+              # partition. Otherwise in case of a window aggregation with VP spanning across
+              # several polls, the data could not be complete.
+              vps.reducer.call(
+                vps.partitioner.call(msg)
+              )
             end
 
             groupings.each do |key, messages_group|

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -22,6 +22,7 @@ module Karafka
             :partitioner,
             :max_partitions,
             :offset_metadata_strategy,
+            :reducer,
             keyword_init: true
           ) { alias_method :active?, :active }
         end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -31,6 +31,7 @@ module Karafka
               nested(:virtual_partitions) do
                 required(:active) { |val| [true, false].include?(val) }
                 required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:reducer) { |val| val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
                 required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
               end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -26,18 +26,24 @@ module Karafka
             # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
             #   offset. `:exact` will match the offset matching metadata and `:current` will select
             #   the most recently reported metadata
+            # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
               partitioner: nil,
-              offset_metadata_strategy: :current
+              offset_metadata_strategy: :current,
+              reducer: nil
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
                 max_partitions: max_partitions,
                 partitioner: partitioner,
-                offset_metadata_strategy: offset_metadata_strategy
+                offset_metadata_strategy: offset_metadata_strategy,
+                # If no reducer provided, we use this one. It just runs a modulo on the sum of
+                # a stringified version, providing fairly good distribution.
+                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions }
               )
             end
 

data/lib/karafka/runner.rb

@@ -3,17 +3,18 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    def initialize
-      @manager = App.config.internal.connection.manager
-      @conductor = App.config.internal.connection.conductor
-    end
+    include Helpers::ConfigImporter.new(
+      manager: %i[internal connection manager],
+      conductor: %i[internal connection conductor],
+      jobs_queue_class: %i[internal processing jobs_queue_class]
+    )
 
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
-      jobs_queue = App.config.internal.processing.jobs_queue_class.new
+      jobs_queue = jobs_queue_class.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
@@ -23,7 +24,7 @@ module Karafka
       Karafka::App.run!
 
       # Register all the listeners so they can be started and managed
-      @manager.register(listeners)
+      manager.register(listeners)
 
       workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
 
@@ -32,10 +33,10 @@ module Karafka
       Karafka::Server.listeners = listeners
       Karafka::Server.jobs_queue = jobs_queue
 
-      until @manager.done?
-        @conductor.wait
+      until manager.done?
+        conductor.wait
 
-        @manager.control
+        manager.control
       end
 
       # We close the jobs queue only when no listener threads are working.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.2'
+  VERSION = '2.4.3'
 end

data/lib/karafka.rb

@@ -92,7 +92,17 @@ module Karafka
     #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
     #   Karafka.boot_file #=> '/home/app_path/app.rb'
     def boot_file
-      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+      boot_file = Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+
+      return boot_file if boot_file.absolute?
+      return boot_file if boot_file.to_s == 'false'
+
+      Pathname.new(
+        File.expand_path(
+          boot_file,
+          Karafka.root
+        )
+      )
     end
 
     # We need to be able to overwrite both monitor and logger after the configuration in case they

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 2.4.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-05-14 00:00:00.000000000 Z
+date: 2024-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 2.7.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 2.7.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -549,7 +549,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.