karafka 2.2.14 → 2.3.0.alpha2

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -25,7 +25,7 @@ module Karafka
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
-          @consumer = Connection::Proxy.new(consumer)
+          @consumer = ::Karafka::Connection::Proxy.new(consumer)
           @expanded_topics = expanded_topics
           @mapped_topics = Hash.new { |h, k| h[k] = {} }
         end

data/lib/karafka/pro/iterator.rb

@@ -22,11 +22,6 @@ module Karafka
     #
     # It does **not** create a consumer group and does not have any offset management.
     class Iterator
-      # Local partition reference for librdkafka
-      Partition = Struct.new(:partition, :offset)
-
-      private_constant :Partition
-
       # A simple API allowing to iterate over topic/partition data, without having to subscribe
       # and deal with rebalances. This API allows for multi-partition streaming and is optimized
       # for data lookups. It allows for explicit stopping iteration over any partition during
@@ -127,7 +122,7 @@ module Karafka
 
         @current_consumer.pause(
           Rdkafka::Consumer::TopicPartitionList.new(
-            name => [Partition.new(partition, 0)]
+            name => [Rdkafka::Consumer::Partition.new(partition, 0)]
           )
         )
       end

data/lib/karafka/pro/loader.rb

@@ -53,6 +53,7 @@ module Karafka
           features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
+          expand
 
           load_topic_features
         end
@@ -82,12 +83,17 @@ module Karafka
 
           icfg.cli.contract = Contracts::ServerCliOptions.new
 
+          # Use manager that supports multiplexing
+          icfg.connection.manager = Connection::Manager.new
+
           icfg.processing.coordinator_class = Processing::Coordinator
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue
+          icfg.processing.executor_class = Processing::Executor
           icfg.processing.jobs_builder = Processing::JobsBuilder.new
           icfg.processing.strategy_selector = Processing::StrategySelector.new
+          icfg.processing.expansions_selector = Processing::ExpansionsSelector.new
 
           icfg.active_job.consumer_class = ActiveJob::Consumer
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
@@ -96,6 +102,14 @@ module Karafka
           config.monitor.subscribe(Instrumentation::PerformanceTracker.instance)
         end
 
+        # Adds extra modules to certain classes
+        # This expands their functionalities with things that are needed when operating in Pro
+        # It is used only when given class is part of the end user API and cannot be swapped by
+        # a pluggable component
+        def expand
+          Karafka::BaseConsumer.include Pro::BaseConsumer
+        end
+
         # Loads the Pro features of Karafka
         # @note Object space lookup is not the fastest but we do it once during boot, so it's ok
         def load_topic_features

data/lib/karafka/pro/processing/coordinator.rb

@@ -42,7 +42,8 @@ module Karafka
 
           @virtual_offset_manager = VirtualOffsetManager.new(
             topic.name,
-            partition
+            partition,
+            topic.virtual_partitions.offset_metadata_strategy
           )
 
           # We register our own "internal" filter to support filtering of messages that were marked

data/lib/karafka/pro/processing/executor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro executor that supports periodic jobs
+      class Executor < Karafka::Processing::Executor
+        # Runs the code that should happen before periodic job is scheduled
+        #
+        # @note While jobs are called `Periodic`, from the consumer perspective it is "ticking".
+        #   This name was taken for a reason: we may want to introduce periodic ticking also not
+        #   only during polling but for example on wait and a name "poll" would not align well.
+        #   A name "periodic" is not a verb and our other consumer actions are verbs like:
+        #   consume or revoked. So for the sake of consistency we have ticking here.
+        def before_schedule_periodic
+          consumer.on_before_schedule_tick
+        end
+
+        # Triggers consumer ticking
+        def periodic
+          consumer.on_tick
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro selector of appropriate topic setup based features enhancements.
+      class ExpansionsSelector < Karafka::Processing::ExpansionsSelector
+        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+        #   expansions
+        # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance
+        #   the consumer
+        def find(topic)
+          # Start with the non-pro expansions
+          expansions = super
+          expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/periodic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Job that represents a "ticking" work. Work that we run periodically for the Periodics
+        # enabled topics.
+        class Periodic < ::Karafka::Processing::Jobs::Base
+          # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
+          #   run a given job
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Code executed before we schedule this job
+          def before_schedule
+            executor.before_schedule_periodic
+          end
+
+          # Runs the executor periodic action
+          def call
+            executor.periodic
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Non-Blocking version of the Periodic job
+        # We use this version for LRJ topics for cases where saturated resources would not allow
+        # to run this job for extended period of time. Under such scenarios, if we would not use
+        # a non-blocking one, we would reach max.poll.interval.ms.
+        class PeriodicNonBlocking < Periodic
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Periodic`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -16,12 +16,12 @@ module Karafka
     module Processing
       # Pro jobs builder that supports lrj
       class JobsBuilder < ::Karafka::Processing::JobsBuilder
-        # @param executor [Karafka::Processing::Executor]
+        # @param executor [Karafka::Pro::Processing::Executor]
         def idle(executor)
           Karafka::Processing::Jobs::Idle.new(executor)
         end
 
-        # @param executor [Karafka::Processing::Executor]
+        # @param executor [Karafka::Pro::Processing::Executor]
         # @param messages [Karafka::Messages::Messages] messages batch to be consumed
         # @return [Karafka::Processing::Jobs::Consume] blocking job
         # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
@@ -33,7 +33,7 @@ module Karafka
           end
         end
 
-        # @param executor [Karafka::Processing::Executor]
+        # @param executor [Karafka::Pro::Processing::Executor]
         # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
         # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
         #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
@@ -44,6 +44,17 @@ module Karafka
             super
           end
         end
+
+        # @param executor [Karafka::Pro::Processing::Executor]
+        # @return [Jobs::Periodic] Periodic job
+        # @return [Jobs::PeriodicNonBlocking] Periodic non-blocking job
+        def periodic(executor)
+          if executor.topic.long_running_job?
+            Jobs::PeriodicNonBlocking.new(executor)
+          else
+            Jobs::Periodic.new(executor)
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/offset_metadata/consumer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Offset Metadata support on the processing side
+      module OffsetMetadata
+        # Extra API methods for offset metadata fetching
+        # @note Part of this feature API is embedded directly into the strategies because it alters
+        #   how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate. Because
+        #   of that, they had to be embedded into the strategies.
+        module Consumer
+          # @param cache [Boolean] should we use cached result if present (true by default)
+          # @return [false, Object] false in case we do not own the partition anymore or
+          #   deserialized metadata based on the deserializer
+          # @note Caching is on as the assumption here is, that most of the time user will be
+          #   interested only in the offset metadata that "came" from the time prior to the
+          #   rebalance. That is because the rest of the metadata (current) is created and
+          #   controlled by the user himself, thus there is no need to retrieve it. In case this
+          #   is not true and user wants to always get the Kafka metadata, `cache` value of this
+          #   feature can be set to false.
+          def offset_metadata(cache: true)
+            return false if revoked?
+
+            Fetcher.find(topic, partition, cache: cache)
+          end
+
+          alias committed_offset_metadata offset_metadata
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/offset_metadata/fetcher.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Offset Metadata support on the processing side
+      module OffsetMetadata
+        # This fetcher is responsible for fetching and caching committed offsets metadata
+        # information.
+        #
+        # By design we fetch all information for a requested topic assignments. Not all topics from
+        # the same subscription group may need metadata and even if, we can run the few smaller
+        # queries. This approach prevents us from querying all assigned topics data in one go
+        # preventing excessive queries.
+        #
+        # Since the assumption is, that user will not have to reach out for the later metadata
+        # since it is produced in the context of a given consumer assignment, we can cache the
+        # initial result and only allow users for explicit invalidation.
+        class Fetcher
+          include Singleton
+
+          class << self
+            extend Forwardable
+
+            def_delegators :instance, :register, :clear, :find
+          end
+
+          def initialize
+            @mutexes = {}
+            @clients = {}
+            @tpls = {}
+          end
+
+          # Registers a client of a given subscription group, so we can use it for queries later on
+          # @param client [Karafka::Connection::Client]
+          # @note Since we store the client reference and not the underlying rdkafka consumer
+          #   instance, we do not have to deal with the recovery as it is abstracted away
+          def register(client)
+            @clients[client.subscription_group] = client
+            # We use one mutex per SG because independent SGs can query in parallel
+            @mutexes[client.subscription_group] = Mutex.new
+            @tpls[client.subscription_group] = {}
+          end
+
+          # Queries or retrieves from cache the given offset metadata for the selected partition
+          #
+          # @param topic [Karafka::Routing::Topic] routing topic with subscription group reference
+          # @param partition [Integer] partition for which we want to get stored offset metadata
+          # @param cache [Boolean] forces explicit query to Kafka when false and cache refresh.
+          #   By default we use the setting from the topic level but this can be overwritten on
+          #   a per request basis if needed.
+          # @return [Object, false] deserialized metadata (string deserializer by default) or
+          #   false in case we were not able to obtain the details because we have lost the
+          #   assignment
+          def find(topic, partition, cache: true)
+            cache = topic.offset_metadata.cache? && cache
+
+            tpls = fetch(topic, cache)
+
+            return false unless tpls
+
+            t_partitions = tpls.fetch(topic.name, [])
+            t_partition = t_partitions.find { |t_p| t_p.partition == partition }
+
+            # If we do not have given topic partition here, it means it is no longer part of our
+            # assignment and we should return false
+            return false unless t_partition
+
+            topic.offset_metadata.deserializer.call(t_partition.metadata)
+          end
+
+          # Clears cache of a given subscription group. It is triggered on assignment changes.
+          #
+          # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group that
+          #   we want to clear.
+          def clear(subscription_group)
+            @mutexes.fetch(subscription_group).synchronize do
+              @tpls[subscription_group].clear
+            end
+          end
+
+          private
+
+          # Fetches from Kafka all committed offsets for the given topic partitions that are
+          # assigned to this process.
+          #
+          # We fetch all because in majority of the cases, the behavior of the end user code is
+          #   not specific to a given partition both same for all. In such cases we save on
+          #   querying as we get all data for all partitions in one go.
+          #
+          # @param topic [Karafka::Routing::Topic] topic for which we want to fetch tpls data
+          # @param cache [Boolean] should we return cached data if present
+          def fetch(topic, cache)
+            subscription_group = topic.subscription_group
+            t_tpls = @tpls.fetch(subscription_group, false)
+            t_tpl = t_tpls[topic]
+
+            return t_tpl if t_tpl && cache
+
+            assigned_tpls = @clients.fetch(subscription_group).assignment
+            t_tpl = assigned_tpls.to_h.fetch(topic.name, false)
+
+            # May be false in case we lost given assignment but still run LRJ
+            return false unless t_tpl
+            return false if t_tpl.empty?
+
+            @mutexes.fetch(subscription_group).synchronize do
+              rd_tpl = Rdkafka::Consumer::TopicPartitionList.new(topic.name => t_tpl)
+
+              # While in theory we could lost assignment while being here, this will work and will
+              # return us proper tpl, we do not deal with this case on this layer and report anyhow
+              # There will not be any exception and this will operate correctly
+              t_tpls[topic] = @clients.fetch(subscription_group).committed(rd_tpl).to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/offset_metadata/listener.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module OffsetMetadata
+        # Keeps track of rebalances and updates the fetcher
+        # Since we cache the tpls with metadata, we need to invalidate them on events that would
+        # cause changes in the assignments
+        class Listener
+          # When we start listening we need to register this client in the metadata fetcher, so
+          # we have the client related to a given subscription group that we can use in fetcher
+          # since fetcher may be used in filtering API and other places outside of the standard
+          # consumer flow
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_before_fetch_loop(event)
+            Fetcher.register event[:client]
+          end
+
+          # Invalidates internal cache when assignments change so we can get correct metadata
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_rebalance_partitions_assigned(event)
+            Fetcher.clear event[:subscription_group]
+          end
+
+          # Invalidates internal cache when assignments change so we can get correct metadata
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_rebalance_partitions_revoked(event)
+            Fetcher.clear event[:subscription_group]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -31,9 +31,19 @@ module Karafka
             @mutex = Mutex.new
           end
 
+          # Schedules any jobs provided in a fifo order
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>]
+          def schedule_fifo(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
           # Runs the consumption jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          # @param jobs_array
+          #   [Array<Karafka::Processing::Jobs::Consume, Processing::Jobs::ConsumeNonBlocking>]
+          #   jobs for scheduling
           def on_schedule_consumption(jobs_array)
             @mutex.synchronize do
               schedule_consumption(jobs_array)
@@ -42,53 +52,59 @@ module Karafka
 
           # Should schedule the consumption jobs
           #
-          # @param _jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          # @param _jobs_array
+          #   [Array<Karafka::Processing::Jobs::Consume, Processing::Jobs::ConsumeNonBlocking>]
+          #   jobs for scheduling
           def schedule_consumption(_jobs_array)
             raise NotImplementedError, 'Implement in a subclass'
           end
 
           # Runs the revocation jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          # @param jobs_array
+          #   [Array<Karafka::Processing::Jobs::Revoked, Processing::Jobs::RevokedNonBlocking>]
+          #   jobs for scheduling
           def on_schedule_revocation(jobs_array)
             @mutex.synchronize do
               schedule_revocation(jobs_array)
             end
           end
 
-          # Schedules the revocation jobs.
-          #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
-          #
-          # @note We provide a default scheduler logic here because by default revocation jobs
-          #   should be scheduled as fast as possible.
-          def schedule_revocation(jobs_array)
-            jobs_array.each do |job|
-              @queue << job
-            end
-          end
-
           # Runs the shutdown jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Shutdown>] jobs for scheduling
           def on_schedule_shutdown(jobs_array)
             @mutex.synchronize do
               schedule_shutdown(jobs_array)
             end
           end
 
-          # Schedules the shutdown jobs.
+          # Runs the idle jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Idle>] jobs for scheduling
+          def on_schedule_idle(jobs_array)
+            @mutex.synchronize do
+              schedule_idle(jobs_array)
+            end
+          end
+
+          # Runs the periodic jobs scheduling flow under a mutex
           #
-          # @note We provide a default scheduler logic here because by default revocation jobs
-          #   should be scheduled as fast as possible.
-          def schedule_shutdown(jobs_array)
-            jobs_array.each do |job|
-              @queue << job
+          # @param jobs_array
+          #   [Array<Processing::Jobs::Periodic, Processing::Jobs::PeriodicNonBlocking>]
+          #   jobs for scheduling
+          def on_schedule_periodic(jobs_array)
+            @mutex.synchronize do
+              schedule_periodic(jobs_array)
             end
           end
 
+          # Schedule by default all except consumption as fifo
+          alias schedule_revocation schedule_fifo
+          alias schedule_shutdown schedule_fifo
+          alias schedule_idle schedule_fifo
+          alias schedule_periodic schedule_fifo
+
           # Runs the manage tick under mutex
           def on_manage
             @mutex.synchronize { manage }

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -31,7 +31,9 @@ module Karafka
         class Default < Base
           # Schedules jobs in the LJF order for consumption
           #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          # @param jobs_array
+          #   [Array<Karafka::Processing::Jobs::Consume, Processing::Jobs::ConsumeNonBlocking>]
+          #   jobs for scheduling
           def on_schedule_consumption(jobs_array)
             perf_tracker = Instrumentation::PerformanceTracker.instance
 
@@ -53,23 +55,19 @@ module Karafka
             end
           end
 
-          # Schedules jobs in the fifo order
-          #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-          def on_schedule_revocation(jobs_array)
+          # Schedules any jobs provided in a fifo order
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>]
+          def schedule_fifo(jobs_array)
             jobs_array.each do |job|
               @queue << job
             end
           end
 
-          # Schedules jobs in the fifo order
-          #
-          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-          def on_schedule_shutdown(jobs_array)
-            jobs_array.each do |job|
-              @queue << job
-            end
-          end
+          # By default all non-consumption work is scheduled in a fifo order
+          alias on_schedule_revocation schedule_fifo
+          alias on_schedule_shutdown schedule_fifo
+          alias on_schedule_idle schedule_fifo
+          alias on_schedule_periodic schedule_fifo
 
           # This scheduler does not have anything to manage as it is a pass through and has no
           # state
@@ -87,7 +85,7 @@ module Karafka
           private
 
           # @param perf_tracker [PerformanceTracker]
-          # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+          # @param job [Karafka::Processing::Jobs::Consume] job we will be processing
           # @return [Numeric] estimated cost of processing this job
           def processing_cost(perf_tracker, job)
             if job.is_a?(::Karafka::Processing::Jobs::Consume)