karafka 2.2.14 → 2.3.0.alpha1

data/lib/karafka/setup/config.rb

@@ -184,8 +184,23 @@ module Karafka
 
         # Namespace for internal connection related settings
         setting :connection do
+          # Manages starting up and stopping Kafka connections
+          setting :manager, default: Connection::Manager.new
+          # Controls frequency of connections management checks
+          setting :conductor, default: Connection::Conductor.new
+
           # Settings that are altered by our client proxy layer
           setting :proxy do
+            # Committed offsets for given CG query
+            setting :committed do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
             # Watermark offsets request settings
             setting :query_watermark_offsets do
               # timeout for this request. For busy or remote clusters, this should be high enough
@@ -222,6 +237,8 @@ module Karafka
           setting :strategy_selector, default: Processing::StrategySelector.new
           # option expansions_selector [Object] processing expansions selector to be used
           setting :expansions_selector, default: Processing::ExpansionsSelector.new
+          # option [Class] executor class
+          setting :executor_class, default: Processing::Executor
         end
 
         # Things related to operating on messages

data/lib/karafka/status.rb

@@ -42,25 +42,34 @@ module Karafka
     end
 
     STATES.each do |state, transition|
-      define_method :"#{state}?" do
-        @status == state
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        def #{state}?
+          @status == :#{state}
+        end
+
+        def #{transition}
+          MUTEX.synchronize do
+            # Do not allow reverse state transitions (we always go one way) or transition to the same
+            # state as currently
+            return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
 
-      define_method transition do
-        MUTEX.synchronize do
-          # Do not allow reverse state transitions (we always go one way) or transition to the same
-          # state as currently
-          return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
+            @status = :#{state}
 
-          @status = state
+            # Skip on creation (initializing)
+            # We skip as during this state we do not have yet a monitor
+            return if initializing?
 
-          # Skip on creation (initializing)
-          # We skip as during this state we do not have yet a monitor
-          return if initializing?
+            # We do not set conductor in the initializer because this status object is created
+            # before the configuration kicks in
+            # We need to signal conductor on each state change as those may be relevant to
+            # listeners operations
+            @conductor ||= Karafka::App.config.internal.connection.conductor
+            @conductor.signal
 
-          Karafka.monitor.instrument("app.#{state}")
+            Karafka.monitor.instrument("app.#{state}")
+          end
         end
-      end
+      RUBY
     end
 
     # @return [Boolean] true if we are in any of the status that would indicate we should no longer

data/lib/karafka/templates/karafka.rb.erb

@@ -68,3 +68,10 @@ class KarafkaApp < Karafka::App
     end
   end
 end
+
+# Karafka now features a Web UI!
+# Visit the setup documentation to get started and enhance your experience.
+#
+# https://karafka.io/docs/Web-UI-Getting-Started
+#
+# Karafka::Web.enable!

data/lib/karafka/time_trackers/partition_usage.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Tracker used to keep time reference when we last time dispatched any job related to a given
+    # topic partition.
+    #
+    # We can use it to know when last time a job was scheduled
+    #
+    # @note We do not track revocation as on revocation we clear given topic partition reference
+    #   not to have a potential memory leak
+    #
+    # @note We do not track shutdown jobs as shutdown is finishing the process, so no time
+    #   sensitive operations remain that would use this
+    #
+    # @note We consider partition as active if we scheduled any job related to it within the tick
+    #   interval. This has nothing to do whether a partition is assigned.
+    class PartitionUsage < Base
+      # Creates new partition usage time tracker
+      def initialize
+        super
+
+        @last_usage = Hash.new do |topics_hash, topic_name|
+          topics_hash[topic_name] = Hash.new do |partitions_hash, partition_id|
+            partitions_hash[partition_id] = 0
+          end
+        end
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @param interval [Integer] minimum interval
+      # @return [Boolean] was this topic partition active
+      def active?(topic, partition, interval)
+        monotonic_now - @last_usage[topic][partition] < interval
+      end
+
+      # Marks usage of given partition
+      #
+      # @param topic [String]
+      # @param partition [Integer]
+      def track(topic, partition)
+        @last_usage[topic][partition] = monotonic_now
+      end
+
+      # Clears references about given partition. Useful on revocation so we do not store old
+      # unassigned partitions data
+      #
+      # @param topic [String]
+      # @param partition [Integer]
+      def revoke(topic, partition)
+        @last_usage[topic].delete(partition)
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.14'
+  VERSION = '2.3.0.alpha1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.14
+  version: 2.3.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-07 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,27 +43,27 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.7
+        version: 2.3.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.7
+        version: 2.3.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: 2.6.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: 2.6.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -123,6 +123,7 @@ files:
 - LICENSE-COMM
 - LICENSE-LGPL
 - README.md
+- SECURITY.md
 - bin/benchmarks
 - bin/create_token
 - bin/integrations
@@ -151,6 +152,7 @@ files:
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/admin.rb
+- lib/karafka/admin/acl.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -162,14 +164,16 @@ files:
 - lib/karafka/cli/server.rb
 - lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/consumer_group_coordinator.rb
+- lib/karafka/connection/conductor.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/listeners_batch.rb
+- lib/karafka/connection/manager.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
 - lib/karafka/connection/proxy.rb
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
+- lib/karafka/connection/status.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
@@ -217,10 +221,13 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/cleaner.rb
 - lib/karafka/pro/cleaner/errors.rb
 - lib/karafka/pro/cleaner/messages/message.rb
 - lib/karafka/pro/cleaner/messages/messages.rb
+- lib/karafka/pro/connection/manager.rb
+- lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
 - lib/karafka/pro/contracts/server_cli_options.rb
 - lib/karafka/pro/encryption.rb
@@ -237,6 +244,8 @@ files:
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/executor.rb
+- lib/karafka/pro/processing/expansions_selector.rb
 - lib/karafka/pro/processing/filters/base.rb
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
@@ -245,9 +254,14 @@ files:
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs/periodic.rb
+- lib/karafka/pro/processing/jobs/periodic_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/jobs_queue.rb
+- lib/karafka/pro/processing/offset_metadata/consumer.rb
+- lib/karafka/pro/processing/offset_metadata/fetcher.rb
+- lib/karafka/pro/processing/offset_metadata/listener.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/schedulers/base.rb
 - lib/karafka/pro/processing/schedulers/default.rb
@@ -328,6 +342,19 @@ files:
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/multiplexing.rb
+- lib/karafka/pro/routing/features/multiplexing/config.rb
+- lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb
+- lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/multiplexing/proxy.rb
+- lib/karafka/pro/routing/features/multiplexing/subscription_group.rb
+- lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb
+- lib/karafka/pro/routing/features/non_blocking_job.rb
+- lib/karafka/pro/routing/features/non_blocking_job/topic.rb
+- lib/karafka/pro/routing/features/offset_metadata.rb
+- lib/karafka/pro/routing/features/offset_metadata/config.rb
+- lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb
+- lib/karafka/pro/routing/features/offset_metadata/topic.rb
 - lib/karafka/pro/routing/features/patterns.rb
 - lib/karafka/pro/routing/features/patterns/builder.rb
 - lib/karafka/pro/routing/features/patterns/config.rb
@@ -343,6 +370,10 @@ files:
 - lib/karafka/pro/routing/features/pausing.rb
 - lib/karafka/pro/routing/features/pausing/contracts/topic.rb
 - lib/karafka/pro/routing/features/pausing/topic.rb
+- lib/karafka/pro/routing/features/periodic_job.rb
+- lib/karafka/pro/routing/features/periodic_job/config.rb
+- lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
+- lib/karafka/pro/routing/features/periodic_job/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contracts/topic.rb
@@ -427,6 +458,7 @@ files:
 - lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb
 - lib/karafka/time_trackers/base.rb
+- lib/karafka/time_trackers/partition_usage.rb
 - lib/karafka/time_trackers/pause.rb
 - lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
@@ -459,7 +491,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # This object represents a collective status of execution of group of listeners running inside
-    # of one consumer group but in separate subscription groups.
-    #
-    # There are cases when we do not want to close a given client when others from the same
-    # consumer group are running because it can cause instabilities due to early shutdown of some
-    # of the clients out of same consumer group.
-    #
-    # We also want to make sure, we close one consumer at a time while others can continue polling.
-    #
-    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
-    # without having a chance to commit changes.
-    class ConsumerGroupCoordinator
-      # @param group_size [Integer] number of separate subscription groups in a consumer group
-      def initialize(group_size)
-        @shutdown_mutex = Mutex.new
-        @group_size = group_size
-        @finished = Set.new
-      end
-
-      # @return [Boolean] true if all the subscription groups from a given consumer group are
-      #   finished
-      def finished?
-        @finished.size == @group_size
-      end
-
-      # @return [Boolean] can we start shutdown on a given listener
-      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
-      def shutdown?
-        finished? && @shutdown_mutex.try_lock
-      end
-
-      # Unlocks the shutdown lock
-      def unlock
-        @shutdown_mutex.unlock if @shutdown_mutex.owned?
-      end
-
-      # Marks given listener as finished
-      # @param listener_id [String]
-      def finish_work(listener_id)
-        @finished << listener_id
-      end
-    end
-  end
-end