karafka 2.3.0.rc1 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b07678937b0dc9c4154c90ccea2e9bbbc25aa2669461d937e859b77e134ddc48
-  data.tar.gz: a2e3eee8a9351a789dc33a1f78a48899a007da29b81205b4383e75a9fa0cc13c
+  metadata.gz: 9d28565a68d29a1f8f14be1878bf3a1f41611028069586fb4ece1572b8eb72eb
+  data.tar.gz: 8c7f6ea1ec67e07ed107c7347d594a26ba923cecc7282c346eb24863e0324412
 SHA512:
-  metadata.gz: d2331d328849f4ffb65aa5deb921bf8d12d732f06b30acc5b17ef1fb844d5da23166dbcd27a6b53a6d57d76c6a7d508e9c5479e064a73aa6e0c2eaed69597c8a
-  data.tar.gz: da8afa92e4b5fc552319b15322b3b3e883c2ec5c2b46ec887d098d34f7a0fbd7a7408558a89d781699eb521309c8fe69d2df7e6a1f46de19259451fb3050e391
+  metadata.gz: 73d43079b8e83babe2f722acdd11a67d6a5c361998ffada9b46b0f28b3ebc9c430d7b28cf56cb2aba808e25c234fffc7b2ee945e9e265160fb7cba6f821fcc70
+  data.tar.gz: 465b1088763ddc3ea63e53ef92765ae5719d6d1a83522288976321d1a64d24616b06d6571ce11425fd8a0dbf36d8c595871b92abfc8de8bcecc444181afe5123

data/CHANGELOG.md

@@ -1,6 +1,9 @@
 # Karafka framework changelog
 
-## 2.3.0 (Unreleased)
+## 2.3.1 (2024-02-08)
+- [Refactor] Ensure that `Karafka::Helpers::Async#async_call` can run from multiple threads.
+
+## 2.3.0 (2024-01-26)
 - **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
 - **[Feature]** Provide ability to multiplex subscription groups (Pro)
 - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
@@ -23,8 +26,13 @@
 - [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
 - [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
 - [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
+- [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
 - [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
 
+### Upgrade Notes
+
+Available [here](https://karafka.io/docs/Upgrades-2.3/).
+
 ## 2.2.14 (2023-12-07)
 - **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
 - **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.

data/Gemfile

@@ -10,7 +10,7 @@ gemspec
 # They are added here because they are part of the integration suite
 group :integrations do
   gem 'activejob'
-  gem 'karafka-web'
+  gem 'karafka-web', '>= 0.8.0'
 end
 
 group :test do

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.rc1)
-      karafka-core (>= 2.3.0.rc1, < 2.4.0)
+    karafka (2.3.1)
+      karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -27,7 +27,7 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.0)
       ruby2_keywords
@@ -39,38 +39,38 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0.rc1)
-      karafka-rdkafka (>= 0.14.7, < 0.15.0)
-    karafka-rdkafka (0.14.7)
+    karafka-core (2.3.0)
+      karafka-rdkafka (>= 0.14.8, < 0.15.0)
+    karafka-rdkafka (0.14.9)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.10)
+    karafka-web (0.8.1)
       erubi (~> 1.4)
-      karafka (>= 2.2.9, < 3.0.0)
-      karafka-core (>= 2.2.4, < 3.0.0)
+      karafka (>= 2.3.0, < 2.4.0)
+      karafka-core (>= 2.3.0, < 2.4.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
     minitest (5.21.2)
     mutex_m (0.2.0)
-    rack (3.0.8)
+    rack (3.0.9)
     rake (13.1.0)
     roda (3.76.0)
       rack
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.0)
     ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
@@ -81,7 +81,7 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.12)
+    waterdrop (2.6.13)
       karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.12)
@@ -95,7 +95,7 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-web
+  karafka-web (>= 0.8.0)
   rspec
   simplecov
 

data/README.md

@@ -57,8 +57,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-# Make sure to install Karafka 2.2
-bundle add karafka --version ">= 2.2.10"
+# Make sure to install Karafka 2.3
+bundle add karafka --version ">= 2.3.0"
 
 bundle exec karafka install
 ```

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
   spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/admin.rb

@@ -191,34 +191,41 @@ module Karafka
         mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
         settings = { 'group.id': mapped_consumer_group_id }
 
-        with_consumer(settings) do |consumer|
-          # If we have any time based stuff to resolve, we need to do it prior to commits
-          unless time_tpl.empty?
-            real_offsets = consumer.offsets_for_times(time_tpl)
-
-            real_offsets.to_h.each do |name, results|
-              results.each do |result|
-                raise(Errors::InvalidTimeBasedOffsetError) unless result
-
-                partition = result.partition
-
-                # Negative offset means we're beyond last message and we need to query for the
-                # high watermark offset to get the most recent offset and move there
-                if result.offset.negative?
-                  _, offset = consumer.query_watermark_offsets(name, result.partition)
-                else
-                  # If we get an offset, it means there existed a message close to this time
-                  # location
-                  offset = result.offset
+        # This error can occur when we query a broker that is not a coordinator because something
+        # was changing in the cluster. We should be able to safely restart our seeking request
+        # when this happens without any issues
+        #
+        # We wrap the consumer creation, so we retry with a new consumer instance
+        with_rdkafka_retry(codes: %i[not_coordinator]) do
+          with_consumer(settings) do |consumer|
+            # If we have any time based stuff to resolve, we need to do it prior to commits
+            unless time_tpl.empty?
+              real_offsets = consumer.offsets_for_times(time_tpl)
+
+              real_offsets.to_h.each do |name, results|
+                results.each do |result|
+                  raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+                  partition = result.partition
+
+                  # Negative offset means we're beyond last message and we need to query for the
+                  # high watermark offset to get the most recent offset and move there
+                  if result.offset.negative?
+                    _, offset = consumer.query_watermark_offsets(name, result.partition)
+                  else
+                    # If we get an offset, it means there existed a message close to this time
+                    # location
+                    offset = result.offset
+                  end
+
+                  # Since now we have proper offsets, we can add this to the final tpl for commit
+                  tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
                 end
-
-                # Since now we have proper offsets, we can add this to the final tpl for commit
-                tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
               end
             end
-          end
 
-          consumer.commit(tpl, false)
+            consumer.commit(tpl, false)
+          end
         end
       end
 

data/lib/karafka/constraints.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to check some constraints that cannot be easily defined by Bundler
+  # At the moment we use it to ensure, that if Karafka is used, it operates with the expected
+  # web ui version and that older versions of Web UI that would not be compatible with the API
+  # changes in karafka are not used.
+  #
+  # We can make Web UI require certain karafka version range, but at the moment we do not have a
+  # strict 1:1 release pattern matching those two.
+  module Constraints
+    class << self
+      # Verifies that optional requirements are met.
+      def verify!
+        # Skip verification if web is not used at all
+        return unless require_version('karafka/web')
+
+        # All good if version higher than 0.7.x because we expect 0.8.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.7.100')
+
+        # If older web-ui used, we cannot allow it
+        raise(
+          Errors::DependencyConstraintsError,
+          'karafka-web < 0.8.0 is not compatible with this karafka version'
+        )
+      end
+
+      private
+
+      # Requires given version file from a gem location
+      # @param version_location [String]
+      # @return [Boolean] true if it was required or false if not reachable
+      def require_version(version_location)
+        require "#{version_location}/version"
+
+        true
+      rescue LoadError
+        false
+      end
+
+      # Builds a version object for comparing
+      # @param string [String]
+      # @return [::Gem::Version]
+      def version(string)
+        ::Gem::Version.new(string)
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -80,5 +80,12 @@ module Karafka
     # Raised in transactions when we attempt to store offset for a partition that we have lost
     # This does not affect producer only transactions, hence we raise it only on offset storage
     AssignmentLostError = Class.new(BaseError)
+
+    # Raised if optional dependencies like karafka-web are required in a version that is not
+    # supported by the current framework version.
+    #
+    # Because we do not want to require web out of the box and we do not want to lock web with
+    # karafka 1:1, we do such a sanity check
+    DependencyConstraintsError = Class.new(BaseError)
   end
 end

data/lib/karafka/helpers/async.rb

@@ -8,6 +8,12 @@ module Karafka
     # @note Thread running code needs to manage it's own exceptions. If they leak out, they will
     #   abort thread on exception.
     module Async
+      # Mutex used to ensure we do not create multiple threads if we decide to run this
+      # in parallel on multiple threads
+      MUTEX = Mutex.new
+
+      private_constant :MUTEX
+
       class << self
         # Adds forwardable to redirect thread-based control methods to the underlying thread that
         # runs the async operations
@@ -22,10 +28,14 @@ module Karafka
 
       # Runs the `#call` method in a new thread
       def async_call
-        @thread = Thread.new do
-          Thread.current.abort_on_exception = true
+        MUTEX.synchronize do
+          return if @thread&.alive?
+
+          @thread = Thread.new do
+            Thread.current.abort_on_exception = true
 
-          call
+            call
+          end
         end
       end
     end

data/lib/karafka/runner.rb

@@ -18,6 +18,10 @@ module Karafka
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
 
+      # We mark it prior to delegating to the manager as manager will have to start at least one
+      # connection to Kafka, hence running
+      Karafka::App.run!
+
       # Register all the listeners so they can be started and managed
       @manager.register(listeners)
 

data/lib/karafka/server.rb

@@ -61,10 +61,9 @@ module Karafka
       end
 
       # Starts Karafka with a supervision
-      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
-      # finish loop (and it won't happen until we explicitly want to stop)
+      # @note We don't need to sleep because Karafka::Runner is locking and waiting to finish loop
+      # (and it won't happen until we explicitly want to stop)
       def start
-        Karafka::App.run!
         Karafka::Runner.new.call
       end
 

data/lib/karafka/templates/karafka.rb.erb

@@ -73,5 +73,3 @@ end
 # Visit the setup documentation to get started and enhance your experience.
 #
 # https://karafka.io/docs/Web-UI-Getting-Started
-#
-# Karafka::Web.enable!

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.rc1'
+  VERSION = '2.3.1'
 end

data/lib/karafka.rb

@@ -143,3 +143,5 @@ end
 
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'
+
+Karafka::Constraints.verify!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.rc1
+  version: 2.3.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-21 00:00:00.000000000 Z
+date: 2024-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -174,6 +174,7 @@ files:
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/connection/status.rb
+- lib/karafka/constraints.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb