karafka 2.3.0.rc1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b07678937b0dc9c4154c90ccea2e9bbbc25aa2669461d937e859b77e134ddc48
-  data.tar.gz: a2e3eee8a9351a789dc33a1f78a48899a007da29b81205b4383e75a9fa0cc13c
+  metadata.gz: bceb8995575bd7c51d4fa6df9905619082080c932b5295749d9562fb53eba2a7
+  data.tar.gz: d16247c466fe08f6a671e48e8b25af7edcf2c32d769a497a46245773a66794de
 SHA512:
-  metadata.gz: d2331d328849f4ffb65aa5deb921bf8d12d732f06b30acc5b17ef1fb844d5da23166dbcd27a6b53a6d57d76c6a7d508e9c5479e064a73aa6e0c2eaed69597c8a
-  data.tar.gz: da8afa92e4b5fc552319b15322b3b3e883c2ec5c2b46ec887d098d34f7a0fbd7a7408558a89d781699eb521309c8fe69d2df7e6a1f46de19259451fb3050e391
+  metadata.gz: 3367c6bace7ab7a1ed7c6ef04e41eefa40c6fef7baf41d0e8e32bb891e1f2799e1564a6ee36b38735b5d514272c88c62e90bcbb54ffa45f175e80b888837d1e5
+  data.tar.gz: 2d31b87c97ce9deb3d47c0332782ad116dbcf4661a7ea55b35ac04a0e87003a8eb73a5b50f7fc9a21eb1ad288eaafff0528b95133fe58f97ada832526959b628

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka framework changelog
 
-## 2.3.0 (Unreleased)
+## 2.3.0 (2024-01-26)
 - **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
 - **[Feature]** Provide ability to multiplex subscription groups (Pro)
 - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
@@ -23,8 +23,13 @@
 - [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
 - [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
 - [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
+- [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
 - [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
 
+### Upgrade Notes
+
+Available [here](https://karafka.io/docs/Upgrades-2.3/).
+
 ## 2.2.14 (2023-12-07)
 - **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
 - **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.

data/Gemfile

@@ -10,7 +10,7 @@ gemspec
 # They are added here because they are part of the integration suite
 group :integrations do
   gem 'activejob'
-  gem 'karafka-web'
+  gem 'karafka-web', '>= 0.8.0.rc1'
 end
 
 group :test do

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.rc1)
-      karafka-core (>= 2.3.0.rc1, < 2.4.0)
+    karafka (2.3.0)
+      karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -39,16 +39,16 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0.rc1)
-      karafka-rdkafka (>= 0.14.7, < 0.15.0)
-    karafka-rdkafka (0.14.7)
+    karafka-core (2.3.0)
+      karafka-rdkafka (>= 0.14.8, < 0.15.0)
+    karafka-rdkafka (0.14.8)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.10)
+    karafka-web (0.8.0.rc1)
       erubi (~> 1.4)
-      karafka (>= 2.2.9, < 3.0.0)
-      karafka-core (>= 2.2.4, < 3.0.0)
+      karafka (>= 2.3.0.rc1, < 2.4.0)
+      karafka-core (>= 2.3.0.rc1, < 2.4.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
@@ -95,7 +95,7 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-web
+  karafka-web (>= 0.8.0.rc1)
   rspec
   simplecov
 

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
   spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/admin.rb

@@ -191,34 +191,41 @@ module Karafka
         mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
         settings = { 'group.id': mapped_consumer_group_id }
 
-        with_consumer(settings) do |consumer|
-          # If we have any time based stuff to resolve, we need to do it prior to commits
-          unless time_tpl.empty?
-            real_offsets = consumer.offsets_for_times(time_tpl)
-
-            real_offsets.to_h.each do |name, results|
-              results.each do |result|
-                raise(Errors::InvalidTimeBasedOffsetError) unless result
-
-                partition = result.partition
-
-                # Negative offset means we're beyond last message and we need to query for the
-                # high watermark offset to get the most recent offset and move there
-                if result.offset.negative?
-                  _, offset = consumer.query_watermark_offsets(name, result.partition)
-                else
-                  # If we get an offset, it means there existed a message close to this time
-                  # location
-                  offset = result.offset
+        # This error can occur when we query a broker that is not a coordinator because something
+        # was changing in the cluster. We should be able to safely restart our seeking request
+        # when this happens without any issues
+        #
+        # We wrap the consumer creation, so we retry with a new consumer instance
+        with_rdkafka_retry(codes: %i[not_coordinator]) do
+          with_consumer(settings) do |consumer|
+            # If we have any time based stuff to resolve, we need to do it prior to commits
+            unless time_tpl.empty?
+              real_offsets = consumer.offsets_for_times(time_tpl)
+
+              real_offsets.to_h.each do |name, results|
+                results.each do |result|
+                  raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+                  partition = result.partition
+
+                  # Negative offset means we're beyond last message and we need to query for the
+                  # high watermark offset to get the most recent offset and move there
+                  if result.offset.negative?
+                    _, offset = consumer.query_watermark_offsets(name, result.partition)
+                  else
+                    # If we get an offset, it means there existed a message close to this time
+                    # location
+                    offset = result.offset
+                  end
+
+                  # Since now we have proper offsets, we can add this to the final tpl for commit
+                  tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
                 end
-
-                # Since now we have proper offsets, we can add this to the final tpl for commit
-                tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
               end
             end
-          end
 
-          consumer.commit(tpl, false)
+            consumer.commit(tpl, false)
+          end
         end
       end
 

data/lib/karafka/constraints.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to check some constraints that cannot be easily defined by Bundler
+  # At the moment we use it to ensure, that if Karafka is used, it operates with the expected
+  # web ui version and that older versions of Web UI that would not be compatible with the API
+  # changes in karafka are not used.
+  #
+  # We can make Web UI require certain karafka version range, but at the moment we do not have a
+  # strict 1:1 release pattern matching those two.
+  module Constraints
+    class << self
+      # Verifies that optional requirements are met.
+      def verify!
+        # Skip verification if web is not used at all
+        return unless require_version('karafka/web')
+
+        # All good if version higher than 0.7.x because we expect 0.8.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.7.100')
+
+        # If older web-ui used, we cannot allow it
+        raise(
+          Errors::DependencyConstraintsError,
+          'karafka-web < 0.8.0 is not compatible with this karafka version'
+        )
+      end
+
+      private
+
+      # Requires given version file from a gem location
+      # @param version_location [String]
+      # @return [Boolean] true if it was required or false if not reachable
+      def require_version(version_location)
+        require "#{version_location}/version"
+
+        true
+      rescue LoadError
+        false
+      end
+
+      # Builds a version object for comparing
+      # @param string [String]
+      # @return [::Gem::Version]
+      def version(string)
+        ::Gem::Version.new(string)
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -80,5 +80,12 @@ module Karafka
     # Raised in transactions when we attempt to store offset for a partition that we have lost
     # This does not affect producer only transactions, hence we raise it only on offset storage
     AssignmentLostError = Class.new(BaseError)
+
+    # Raised if optional dependencies like karafka-web are required in a version that is not
+    # supported by the current framework version.
+    #
+    # Because we do not want to require web out of the box and we do not want to lock web with
+    # karafka 1:1, we do such a sanity check
+    DependencyConstraintsError = Class.new(BaseError)
   end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.rc1'
+  VERSION = '2.3.0'
 end

data/lib/karafka.rb

@@ -143,3 +143,5 @@ end
 
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'
+
+Karafka::Constraints.verify!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.rc1
+  version: 2.3.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-21 00:00:00.000000000 Z
+date: 2024-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -174,6 +174,7 @@ files:
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/connection/status.rb
+- lib/karafka/constraints.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb