karafka 2.4.0.beta1 → 2.4.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c1493774aae6fa07d816d8e76b9d5577f5483e9d93301dc10a8e7d7a86c7045
-  data.tar.gz: 32251813e9e3b9a9f7b75554a7b0175c2e1cf30456acb3207464a36587073259
+  metadata.gz: 0d682c1f18dd2c0bba4048462095d2129f7758bcbafb810a52f3e2e1515c5e47
+  data.tar.gz: bd4c87fda52f44b61cbcb579369343b327ce04b2229e0903642a6c5a537072eb
 SHA512:
-  metadata.gz: 614cc7507dd865a8efde56e40a77f694efb268c4a877917834a7e23d78175cb9ddb4574ccaac75db86fea1e1de1eae17f4df694251268d90f2822a4868cddf0d
-  data.tar.gz: f6ec801b2e47e531bcd4be2fd6e9e8bc01837b6218ffea1b3e187edc450fa519d2fb5f0fa8a69f5b0e67aabff8496aac1dbbc156bb45c2dd02b1ea495fb5710f
+  metadata.gz: 5ced4aea757aa2bda21ac9a3519344389a7f03a3229b849e3c05fce262494cd3c8f58c1544f1c94fab02559def0f5b1c4c2165933bef6af8559a3b9a19e86299
+  data.tar.gz: c5f4a61cd58554d62fd9ce48a627d8bc4266eb4a244d08ec7ed9ca04006c2e19f113698262d09caa743c8c6896c49666d454fb93b7fdf81d17c3c9ce4a561dd8

data/CHANGELOG.md

@@ -16,6 +16,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - **[Feature]** Introduce `karafka topics plan` to describe changes that will be applied when migrating.
 - **[Feature]** Introduce ability to use custom message key deserializers.
 - **[Feature]** Introduce ability to use custom message headers deserializers.
+- **[Feature]** Provide `Karafka::Admin::Configs` API for cluster and topics configuration management.
 - [Enhancement] Assign names to internal threads for better debuggability when on `TTIN`.
 - [Enhancement] Provide `log_polling` setting to the `Karafka::Instrumentation::LoggerListener` to silence polling in any non-debug mode.
 - [Enhancement] Provide `metadata#message` to be able to retrieve message from metadata.
@@ -50,6 +51,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - [Fix] Re-raise critical supervisor errors before shutdown.
 - [Fix] Fix a case when right-open (infinite) swarm matching would not pass validations.
 - [Fix] Make `#enqueue_all` output compatible with `ActiveJob.perform_all_later` (oozzal)
+- [Fix] Seek consumer group on a topic level is updating only recent partition.
 
 ### Upgrade Notes
 

data/Gemfile

@@ -11,8 +11,8 @@ gemspec
 # Since some of those are only needed for some specs, they should never be required automatically
 group :integrations do
   gem 'activejob', require: false
-  gem 'karafka-testing', '>= 2.4.0.beta1', require: false
-  gem 'karafka-web', require: false
+  gem 'karafka-testing', '>= 2.4.0.rc1', require: false
+  gem 'karafka-web', '>= 0.9.0.rc1', require: false
   gem 'rspec', require: false
 end
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.0.beta1)
-      karafka-core (>= 2.4.0.rc1, < 2.5.0)
-      waterdrop (>= 2.7.0.rc1, < 3.0.0)
+    karafka (2.4.0.rc1)
+      karafka-core (>= 2.4.0.rc2, < 2.5.0)
+      waterdrop (>= 2.7.0.rc2, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -38,27 +38,27 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0.rc1)
-      karafka-rdkafka (>= 0.15.0.rc1, < 0.16.0)
-    karafka-rdkafka (0.15.0.rc1)
+    karafka-core (2.4.0.rc2)
+      karafka-rdkafka (>= 0.15.0.rc2, < 0.16.0)
+    karafka-rdkafka (0.15.0.rc2)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.0.beta1)
-      karafka (>= 2.3.0, < 2.5.0)
-      waterdrop (>= 2.7.0.beta2)
-    karafka-web (0.8.2)
+    karafka-testing (2.4.0.rc1)
+      karafka (>= 2.4.0.beta1, < 2.5.0)
+      waterdrop (>= 2.7.0.rc1)
+    karafka-web (0.9.0.rc2)
       erubi (~> 1.4)
-      karafka (>= 2.3.0, < 2.4.0)
-      karafka-core (>= 2.3.0, < 2.4.0)
+      karafka (>= 2.4.0.beta1, < 2.5.0)
+      karafka-core (>= 2.4.0.rc1, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.6)
     minitest (5.22.3)
     mutex_m (0.2.0)
     rack (3.0.10)
-    rake (13.1.0)
-    roda (3.78.0)
+    rake (13.2.1)
+    roda (3.79.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -82,8 +82,8 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.0.rc1)
-      karafka-core (>= 2.4.0.rc1, < 3.0.0)
+    waterdrop (2.7.0.rc2)
+      karafka-core (>= 2.4.0.rc2, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.13)
 
@@ -96,10 +96,10 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-testing (>= 2.4.0.beta1)
-  karafka-web
+  karafka-testing (>= 2.4.0.rc1)
+  karafka-web (>= 0.9.0.rc1)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.6
+   2.5.7

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.0
+    image: confluentinc/cp-kafka:7.6.1
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.4.0.rc1', '< 2.5.0'
-  spec.add_dependency 'waterdrop', '>= 2.7.0.rc1', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.0.rc2', '< 2.5.0'
+  spec.add_dependency 'waterdrop', '>= 2.7.0.rc2', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

data/lib/karafka/admin.rb

@@ -183,7 +183,13 @@ module Karafka
         tpl_base.each do |topic, partitions_with_offsets|
           partitions_with_offsets.each do |partition, offset|
             target = offset.is_a?(Time) ? time_tpl : tpl
-            target.add_topic_and_partitions_with_offsets(topic, [[partition, offset]])
+            # We reverse and uniq to make sure that potentially duplicated references are removed
+            # in such a way that the newest stays
+            target.to_h[topic] ||= []
+            target.to_h[topic] << Rdkafka::Consumer::Partition.new(partition, offset)
+            target.to_h[topic].reverse!
+            target.to_h[topic].uniq!(&:partition)
+            target.to_h[topic].reverse!
           end
         end
 
@@ -211,7 +217,11 @@ module Karafka
                 end
 
                 # Since now we have proper offsets, we can add this to the final tpl for commit
-                tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
+                tpl.to_h[name] ||= []
+                tpl.to_h[name] << Rdkafka::Consumer::Partition.new(partition, offset)
+                tpl.to_h[name].reverse!
+                tpl.to_h[name].uniq!(&:partition)
+                tpl.to_h[name].reverse!
               end
             end
           end

data/lib/karafka/constraints.rb

@@ -15,13 +15,13 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.8.1 because we expect 0.8.2 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.8.2')
+        # All good if version higher than 0.9.0.rc1 because we expect 0.9.0.rc1 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc1')
 
         # If older web-ui used, we cannot allow it
         raise(
           Errors::DependencyConstraintsError,
-          'karafka-web < 0.8.2 is not compatible with this karafka version'
+          'karafka-web < 0.9.0 is not compatible with this karafka version'
         )
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.0.beta1'
+  VERSION = '2.4.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.0.beta1
+  version: 2.4.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-04-10 00:00:00.000000000 Z
+date: 2024-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.rc1
+        version: 2.4.0.rc2
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.rc1
+        version: 2.4.0.rc2
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -63,7 +63,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0.rc1
+        version: 2.7.0.rc2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0.rc1
+        version: 2.7.0.rc2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0