karafka 2.4.0 → 2.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8111bc347c891837ee1f7cbabd4d98ee1c55f2bb7366e72aa077628cc4052c14
-  data.tar.gz: 9a616eb14fbbddc433e56691540613a728cab6163632981de1c9f21abee8f0b4
+  metadata.gz: 7f3d5fa334ede552c2b0097acf16a942318ef648bcdbdcb0e84ffefcd5e65772
+  data.tar.gz: d800e8e25a62df01000dee9e3ce5904f3ddf632f66ac49e0c71dde3a6702209b
 SHA512:
-  metadata.gz: 35e52c0f5ffeace79cc58e6ef85918f6ff32c80233419dec83a5b99485cd6f5ac2b163f3d0a73e96b4ac424e27a81a3f8f848d3bed5e461b69eb28399f102d3c
-  data.tar.gz: 544848837b53a5e0e64b9c842f8dadc63a486b784660e37a6f44a67947682c59cd112e6ac509e19de99b7b084402b38d7e00d7fff52a89c9e07172f593b3032e
+  metadata.gz: ac438f23ca449023ff9537a5aa007879f2420dfdfe1cd81c1715322cfea53af8112cdf77663547155a40ba4e154cbc8e9c81df23dff9a6f843bb0601395b7271
+  data.tar.gz: 3e267b623cee9b649182ac08d34a68780e696e9ba24fda448caecb762ebf726f436b1b5b3415240f8409b4c7b66fc62c39c319c58128e429580f3468db1e152e

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka framework changelog
 
+## 2.4.1 (2024-05-10)
+- [Enhancement] Allow for usage of producer variants and alternative producers with ActiveJob Jobs (Pro).
+- [Enhancement] Support `:earliest` and `:latest` in `Karafka::Admin#seek_consumer_group`.
+- [Enhancement] Align configuration attributes mapper with exact librdkafka version used and not master.
+- [Maintenance] Use `base64` from RubyGems as it will no longer be part of standard library in Ruby 3.4.
+- [Fix] Support migrating via aliases and plan with aliases usage.
+- [Fix] Active with default set to `false` cannot be overwritten
+- [Fix] Fix inheritance of ActiveJob adapter `karafka_options` partitioner and dispatch method.
+
 ## 2.4.0 (2024-04-26)
 
 This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.

data/Gemfile

@@ -11,8 +11,8 @@ gemspec
 # Since some of those are only needed for some specs, they should never be required automatically
 group :integrations do
   gem 'activejob', require: false
-  gem 'karafka-testing', '>= 2.4.0.rc1', require: false
-  gem 'karafka-web', '>= 0.9.0.rc3', require: false
+  gem 'karafka-testing', '>= 2.4.0', require: false
+  gem 'karafka-web', '>= 0.9.0', require: false
   gem 'rspec', require: false
 end
 

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.0)
+    karafka (2.4.1)
+      base64 (~> 0.2)
       karafka-core (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -23,7 +24,7 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
@@ -36,27 +37,27 @@ GEM
     ffi (1.16.3)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    i18n (1.14.4)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.0)
       karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.0)
+    karafka-rdkafka (0.15.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.0.rc2)
-      karafka (>= 2.4.0.beta1, < 2.5.0)
-      waterdrop (>= 2.7.0.rc1)
-    karafka-web (0.9.0.rc3)
+    karafka-testing (2.4.3)
+      karafka (>= 2.4.0, < 2.5.0)
+      waterdrop (>= 2.7.0)
+    karafka-web (0.9.1)
       erubi (~> 1.4)
-      karafka (>= 2.4.0.rc1, < 2.5.0)
-      karafka-core (>= 2.4.0.rc2, < 2.5.0)
+      karafka (>= 2.4.0, < 2.5.0)
+      karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.6)
     minitest (5.22.3)
     mutex_m (0.2.0)
-    rack (3.0.10)
+    rack (3.0.11)
     rake (13.2.1)
     roda (3.79.0)
       rack
@@ -69,7 +70,7 @@ GEM
     rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
@@ -82,8 +83,9 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.0)
+    waterdrop (2.7.2)
       karafka-core (>= 2.4.0, < 3.0.0)
+      karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.13)
 
@@ -96,8 +98,8 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-testing (>= 2.4.0.rc1)
-  karafka-web (>= 0.9.0.rc3)
+  karafka-testing (>= 2.4.0)
+  karafka-web (>= 0.9.0)
   rspec
   simplecov
 

data/karafka.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
+  spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
   spec.add_dependency 'waterdrop', '>= 2.7.0', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/karafka/active_job/job_extensions.rb

@@ -23,6 +23,9 @@ module Karafka
         # them
         App.config.internal.active_job.job_options_contract.validate!(new_options)
 
+        # We need to modify this hash because otherwise we would modify parent hash.
+        self._karafka_options = _karafka_options.dup
+
         new_options.each do |name, value|
           _karafka_options[name] = value
         end

data/lib/karafka/admin.rb

@@ -10,6 +10,12 @@ module Karafka
   #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
   #   there is no multi-cluster runtime support.
   module Admin
+    # More or less number of seconds of 1 hundred years
+    # Used for time referencing that does not have to be accurate but needs to be big
+    HUNDRED_YEARS = 100 * 365.25 * 24 * 60 * 60
+
+    private_constant :HUNDRED_YEARS
+
     class << self
       # Allows us to read messages from the topic
       #
@@ -156,6 +162,18 @@ module Karafka
       #
       # @example Move offset to 5 seconds ago on partition 2
       #   Karafka::Admin.seek_consumer_group('group-id', { 'topic' => { 2 => 5.seconds.ago } })
+      #
+      # @example Move to the earliest offset on all the partitions of a topic
+      #   Karafka::Admin.seek_consumer_group('group-id', { 'topic' => 'earliest' })
+      #
+      # @example Move to the latest (high-watermark) offset on all the partitions of a topic
+      #   Karafka::Admin.seek_consumer_group('group-id', { 'topic' => 'latest' })
+      #
+      # @example Move offset of a single partition to earliest
+      #   Karafka::Admin.seek_consumer_group('group-id', { 'topic' => { 1 => 'earliest' } })
+      #
+      # @example Move offset of a single partition to latest
+      #   Karafka::Admin.seek_consumer_group('group-id', { 'topic' => { 1 => 'latest' } })
       def seek_consumer_group(consumer_group_id, topics_with_partitions_and_offsets)
         tpl_base = {}
 
@@ -174,6 +192,32 @@ module Karafka
           end
         end
 
+        tpl_base.each_value do |partitions|
+          partitions.transform_values! do |position|
+            # Support both symbol and string based references
+            casted_position = position.is_a?(Symbol) ? position.to_s : position
+
+            # This remap allows us to transform some special cases in a reference that can be
+            # understood by Kafka
+            case casted_position
+            # Earliest is not always 0. When compacting/deleting it can be much later, that's why
+            # we fetch the oldest possible offset
+            when 'earliest'
+              Time.now - HUNDRED_YEARS
+            # Latest will always be the high-watermark offset and we can get it just by getting
+            # a future position
+            when 'latest'
+              Time.now + HUNDRED_YEARS
+            # Same as `'latest'`
+            when false
+              Time.now - HUNDRED_YEARS
+            # Regular offset case
+            else
+              position
+            end
+          end
+        end
+
         tpl = Rdkafka::Consumer::TopicPartitionList.new
         # In case of time based location, we need to to a pre-resolution, that's why we keep it
         # separately

data/lib/karafka/cli/topics/align.rb

@@ -89,9 +89,19 @@ module Karafka
             desired_configs.transform_keys!(&:to_s)
 
             topic_with_configs.configs.each do |config|
-              next unless desired_configs.key?(config.name)
+              names = config.synonyms.map(&:name) << config.name
 
-              desired_config = desired_configs.fetch(config.name)
+              # We move forward only if given topic config is for altering
+              next if (desired_configs.keys & names).empty?
+
+              desired_config = nil
+
+              # We then find last defined value in our configs for a given attribute
+              # Since attributes can have synonyms, we select last one, which will represent the
+              # last defined value in case someone defined same multiple times
+              desired_configs.each do |name, value|
+                desired_config = value if names.include?(name)
+              end
 
               # Do not migrate if existing and desired values are the same
               next if desired_config == config.value

data/lib/karafka/cli/topics/plan.rb

@@ -143,14 +143,25 @@ module Karafka
               }
 
               scoped = @topics_to_alter[declarative][declarative_name]
+              # declarative name can be a synonym. In such cases we remap it during the discovery
+              # below
+              final_name = declarative_name
 
               topic_c.configs.each do |config|
-                next unless declarative_name == config.name
+                names = config.synonyms.map(&:name) << config.name
 
+                next unless names.include?(declarative_name)
+
+                # Always use a non-synonym name if differs
+                final_name = config.name
                 scoped[:action] = :change
                 scoped[:from] = config.value
               end
 
+              # Aligns the name in case synonym was used
+              target = @topics_to_alter[declarative].delete(declarative_name)
+              @topics_to_alter[declarative][final_name] = target
+
               # Remove change definitions that would migrate to the same value as present
               @topics_to_alter[declarative].delete_if do |_name, details|
                 details[:from] == details[:to]

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -28,14 +28,23 @@ module Karafka
           # each job.
           partitioner: nil,
           # Allows for usage of `:key` or `:partition_key`
-          partition_key_type: :key
+          partition_key_type: :key,
+          # Allows for setting a callable producer since at the moment of defining the class,
+          # variants may not be available
+          #
+          # We do not initialize it with `-> { ::Karafka.producer }` so we do not have to call it
+          #   each time for the defaults to preserve CPU cycles.
+          #
+          # We also do **not** cache the execution of this producer lambda because we want to
+          # support job args based producer selection
+          producer: nil
         }.freeze
 
         private_constant :DEFAULTS
 
         # @param job [ActiveJob::Base] job
         def dispatch(job)
-          ::Karafka.producer.public_send(
+          producer(job).public_send(
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
@@ -47,27 +56,46 @@ module Karafka
         # Bulk dispatches multiple jobs using the Rails 7.1+ API
         # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
         def dispatch_many(jobs)
-          dispatches = Hash.new { |hash, key| hash[key] = [] }
+          # First level is type of dispatch and second is the producer we want to use to dispatch
+          dispatches = Hash.new do |hash, key|
+            hash[key] = Hash.new do |hash2, key2|
+              hash2[key2] = []
+            end
+          end
 
           jobs.each do |job|
             d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+            producer = producer(job)
 
-            dispatches[d_method] << dispatch_details(job).merge!(
+            dispatches[d_method][producer] << dispatch_details(job).merge!(
               topic: job.queue_name,
               payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           end
 
-          dispatches.each do |type, messages|
-            ::Karafka.producer.public_send(
-              type,
-              messages
-            )
+          dispatches.each do |d_method, producers|
+            producers.each do |producer, messages|
+              producer.public_send(
+                d_method,
+                messages
+              )
+            end
           end
         end
 
         private
 
+        # Selects the producer based on options. If callable `:producer` is defined, it will use
+        # it. If not, will just use the `Karafka.producer`.
+        #
+        # @param job [ActiveJob::Base] job instance
+        # @return [WaterDrop::Producer, WaterDrop::Producer::Variant] producer or a variant
+        def producer(job)
+          dynamic_producer = fetch_option(job, :producer, DEFAULTS)
+
+          dynamic_producer ? dynamic_producer.call(job) : ::Karafka.producer
+        end
+
         # @param job [ActiveJob::Base] job instance
         # @return [Hash] hash with dispatch details to which we merge topic and payload
         def dispatch_details(job)

data/lib/karafka/routing/topic.rb

@@ -81,7 +81,16 @@ module Karafka
       # Allows to disable topic by invoking this method and setting it to `false`.
       # @param active [Boolean] should this topic be consumed or not
       def active(active)
-        @active = active
+        # Do not allow for active overrides. Basically if this is set on the topic level, defaults
+        # will not overwrite it and this is desired. Otherwise because of the fact that this is
+        # not a full feature config but just a flag, default value would always overwrite the
+        # per-topic config since defaults application happens after the topic config block
+        unless @active_assigned
+          @active = active
+          @active_assigned = true
+        end
+
+        @active
       end
 
       # @return [Class] consumer class that we should use

data/lib/karafka/setup/attributes_map.rb

@@ -52,9 +52,7 @@ module Karafka
         fetch.wait.max.ms
         group.id
         group.instance.id
-        group.protocol
         group.protocol.type
-        group.remote.assignor
         heartbeat.interval.ms
         interceptors
         internal.termination.signal
@@ -153,6 +151,7 @@ module Karafka
       # List of rdkafka producer accepted attributes
       PRODUCER = %i[
         acks
+        allow.auto.create.topics
         api.version.fallback.ms
         api.version.request
         api.version.request.timeout.ms
@@ -289,7 +288,12 @@ module Karafka
       ].freeze
 
       # Location of the file with rdkafka settings list
-      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+      SOURCE = <<~SOURCE.delete("\n").gsub(/\s+/, '/')
+        https://raw.githubusercontent.com
+          confluentinc/librdkafka
+          v#{Rdkafka::LIBRDKAFKA_VERSION}
+          CONFIGURATION.md
+      SOURCE
 
       private_constant :SOURCE
 
@@ -337,9 +341,6 @@ module Karafka
             end
           end
 
-          # This can be removed when 0.13 librdkafka is released
-          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
-
           attributes.transform_values!(&:sort)
           attributes.each_value { |vals| vals.map!(&:to_sym) }
           attributes

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.0'
+  VERSION = '2.4.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.4.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,8 +35,22 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-04-26 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: karafka-core
   requirement: !ruby/object:Gem::Requirement