karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/encryption.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Out of the box encryption engine for both Karafka and WaterDrop
+    # It uses asymmetric encryption via RSA. We use asymmetric so we can have producers that won't
+    # have ability (when private key not added) to decrypt messages.
+    module Encryption
+      class << self
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:encryption, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          Encryption::Contracts::Config.new.validate!(config.to_h)
+
+          # Don't inject extra components if encryption is not active
+          return unless config.encryption.active
+
+          # This parser is encryption aware
+          config.internal.messages.parser = Messages::Parser.new
+
+          # Encryption for WaterDrop
+          config.producer.middleware.append(Messages::Middleware.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator/expander.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    class Iterator
+      # There are various ways you can provide topics information for iterating.
+      #
+      # This mapper normalizes this data, resolves offsets and maps the time based offsets into
+      # appropriate once
+      #
+      # Following formats are accepted:
+      #
+      # - 'topic1' - just a string with one topic name
+      # - ['topic1', 'topic2'] - just the names
+      # - { 'topic1' => -100 } - names with negative lookup offset
+      # - { 'topic1' => { 0 => 5 } } - names with exact partitions offsets
+      # - { 'topic1' => { 0 => -5 }, 'topic2' => { 1 => 5 } } - with per partition negative offsets
+      # - { 'topic1' => 100 } - means we run all partitions from the offset 100
+      # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
+      # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
+      #
+      class Expander
+        # Expands topics to which we want to subscribe with partitions information in case this
+        # info is not provided.
+        #
+        # @param topics [Array, Hash, String] topics definitions
+        # @return [Hash] expanded and normalized requested topics and partitions data
+        def call(topics)
+          expanded = Hash.new { |h, k| h[k] = {} }
+
+          normalize_format(topics).map do |topic, details|
+            if details.is_a?(Hash)
+              details.each do |partition, offset|
+                expanded[topic][partition] = offset
+              end
+            else
+              partition_count(topic).times do |partition|
+                # If no offsets are provided, we just start from zero
+                expanded[topic][partition] = details || 0
+              end
+            end
+          end
+
+          expanded
+        end
+
+        private
+
+        # Input can be provided in multiple formats. Here we normalize it to one (hash).
+        #
+        # @param topics [Array, Hash, String] requested topics
+        # @return [Hash] normalized hash with topics data
+        def normalize_format(topics)
+          # Simplification for the single topic case
+          topics = [topics] if topics.is_a?(String)
+
+          # If we've got just array with topics, we need to convert that into a representation
+          # that we can expand with offsets
+          topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
+          # We remap by creating new hash, just in case the hash came as the argument for this
+          # expanded. We do not want to modify user provided hash
+          topics.transform_keys(&:to_s)
+        end
+
+        # List of topics with their partition information for expansion
+        # We cache it so we do not have to run consecutive requests to obtain data about multiple
+        # topics
+        def topics
+          @topics ||= Admin.cluster_info.topics
+        end
+
+        # @param name [String] topic name
+        # @return [Integer] number of partitions of the topic we want to iterate over
+        def partition_count(name)
+          topics
+            .find { |topic| topic.fetch(:topic_name) == name }
+            .tap { |topic| topic || raise(Errors::TopicNotFoundError, name) }
+            .fetch(:partitions)
+            .count
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    class Iterator
+      # Because we have various formats in which we can provide the offsets, before we can
+      # subscribe to them, there needs to be a bit of normalization.
+      #
+      # For some of the cases, we need to go to Kafka and get the real offsets or watermarks.
+      #
+      # This builder resolves that and builds a tpl to which we can safely subscribe the way
+      # we want it.
+      class TplBuilder
+        # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
+        # @param expanded_topics [Hash] hash with expanded and normalized topics data
+        def initialize(consumer, expanded_topics)
+          @consumer = Connection::Proxy.new(consumer)
+          @expanded_topics = expanded_topics
+          @mapped_topics = Hash.new { |h, k| h[k] = {} }
+        end
+
+        # @return [Rdkafka::Consumer::TopicPartitionList] final tpl we can use to subscribe
+        def call
+          resolve_partitions_without_offsets
+          resolve_partitions_with_exact_offsets
+          resolve_partitions_with_negative_offsets
+          resolve_partitions_with_time_offsets
+
+          # Final tpl with all the data
+          tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          @mapped_topics.each do |name, partitions|
+            tpl.add_topic_and_partitions_with_offsets(name, partitions)
+          end
+
+          tpl
+        end
+
+        private
+
+        # First we expand on those partitions that do not have offsets defined.
+        # When we operate in case like this, we just start from beginning
+        def resolve_partitions_without_offsets
+          @expanded_topics.each do |name, partitions|
+            # We can here only about the case where we have partitions without offsets
+            next unless partitions.is_a?(Array) || partitions.is_a?(Range)
+
+            # When no offsets defined, we just start from zero
+            @mapped_topics[name] = partitions.map { |partition| [partition, 0] }.to_h
+          end
+        end
+
+        # If we get exact numeric offsets, we can just start from them without any extra work
+        def resolve_partitions_with_exact_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Skip negative and time based offsets
+              next unless offset.is_a?(Integer) && offset >= 0
+
+              # Exact offsets can be used as they are
+              # No need for extra operations
+              @mapped_topics[name][partition] = offset
+            end
+          end
+        end
+
+        # If the offsets are negative, it means we want to fetch N last messages and we need to
+        # figure out the appropriate offsets
+        #
+        # We do it by getting the watermark offsets and just calculating it. This means that for
+        # heavily compacted topics, this may return less than the desired number but it is a
+        # limitation that is documented.
+        def resolve_partitions_with_negative_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Care only about numerical offsets
+              #
+              # For time based we already resolve them via librdkafka lookup API
+              next unless offset.is_a?(Integer)
+
+              low_offset, high_offset = @consumer.query_watermark_offsets(name, partition)
+
+              # Care only about negative offsets (last n messages)
+              #
+              # We reject the above results but we **NEED** to run the `#query_watermark_offsets`
+              # for each topic partition nonetheless. Without this, librdkafka fetches a lot more
+              # metadata about each topic and each partition and this takes much more time than
+              # just getting watermarks. If we do not run watermark, at least an extra second
+              # is added at the beginning of iterator flow
+              #
+              # This may not be significant when this runs in the background but in case of
+              # using iterator in thins like Puma, it heavily impacts the end user experience
+              next unless offset.negative?
+
+              # We add because this offset is negative
+              @mapped_topics[name][partition] = [high_offset + offset, low_offset].max
+            end
+          end
+        end
+
+        # For time based offsets we first need to aggregate them and request the proper offsets.
+        # We want to get all times in one go for all tpls defined with times, so we accumulate
+        # them here and we will make one sync request to kafka for all.
+        def resolve_partitions_with_time_offsets
+          time_tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          # First we need to collect the time based once
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            time_based = {}
+
+            partitions.each do |partition, offset|
+              next unless offset.is_a?(Time)
+
+              time_based[partition] = offset
+            end
+
+            next if time_based.empty?
+
+            time_tpl.add_topic_and_partitions_with_offsets(name, time_based)
+          end
+
+          # If there were no time-based, no need to query Kafka
+          return if time_tpl.empty?
+
+          real_offsets = @consumer.offsets_for_times(time_tpl)
+
+          real_offsets.to_h.each do |name, results|
+            results.each do |result|
+              raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+              @mapped_topics[name][result.partition] = result.offset
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Topic iterator allows you to iterate over topic/partition data and perform lookups for
+    # information that you need.
+    #
+    # It supports early stops on finding the requested data and allows for seeking till
+    # the end. It also allows for signaling, when a given message should be last out of certain
+    # partition, but we still want to continue iterating in other messages.
+    #
+    # It does **not** create a consumer group and does not have any offset management.
+    class Iterator
+      # Local partition reference for librdkafka
+      Partition = Struct.new(:partition, :offset)
+
+      private_constant :Partition
+
+      # A simple API allowing to iterate over topic/partition data, without having to subscribe
+      # and deal with rebalances. This API allows for multi-partition streaming and is optimized
+      # for data lookups. It allows for explicit stopping iteration over any partition during
+      # the iteration process, allowing for optimized lookups.
+      #
+      # @param topics [Array<String>, Hash] list of strings if we want to subscribe to multiple
+      #   topics and all of their partitions or a hash where keys are the topics and values are
+      #   hashes with partitions and their initial offsets.
+      # @param settings [Hash] extra settings for the consumer. Please keep in mind, that if
+      #   overwritten, you may want to include `auto.offset.reset` to match your case.
+      # @param yield_nil [Boolean] should we yield also `nil` values when poll returns nothing.
+      #   Useful in particular for long-living iterators.
+      # @param max_wait_time [Integer] max wait in ms when iterator did not receive any messages
+      #
+      # @note It is worth keeping in mind, that this API also needs to operate within
+      #   `max.poll.interval.ms` limitations on each iteration
+      #
+      # @note In case of a never-ending iterator, you need to set `enable.partition.eof` to `false`
+      #   so we don't stop polling data even when reaching the end (end on a given moment)
+      def initialize(
+        topics,
+        settings: { 'auto.offset.reset': 'beginning' },
+        yield_nil: false,
+        max_wait_time: 200
+      )
+        @topics_with_partitions = Expander.new.call(topics)
+
+        @routing_topics = @topics_with_partitions.map do |name, _|
+          [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
+        end.to_h
+
+        @total_partitions = @topics_with_partitions.map(&:last).sum(&:count)
+
+        @stopped_partitions = 0
+
+        @settings = settings
+        @yield_nil = yield_nil
+        @max_wait_time = max_wait_time
+      end
+
+      # Iterates over requested topic partitions and yields the results with the iterator itself
+      # Iterator instance is yielded because one can run `stop_partition` to stop iterating over
+      # part of data. It is useful for scenarios where we are looking for some information in all
+      # the partitions but once we found it, given partition data is no longer needed and would
+      # only eat up resources.
+      def each
+        Admin.with_consumer(@settings) do |consumer|
+          tpl = TplBuilder.new(consumer, @topics_with_partitions).call
+          consumer.assign(tpl)
+
+          # We need this for self-referenced APIs like pausing
+          @current_consumer = consumer
+
+          # Stream data until we reach the end of all the partitions or until the end user
+          # indicates that they are done
+          until done?
+            message = poll
+
+            # Skip nils if not explicitly required
+            next if message.nil? && !@yield_nil
+
+            if message
+              @current_message = build_message(message)
+
+              yield(@current_message, self)
+            else
+              yield(nil, self)
+            end
+          end
+
+          @current_message = nil
+          @current_consumer = nil
+        end
+
+        # Reset so we can use the same iterator again if needed
+        @stopped_partitions = 0
+      end
+
+      # Stops the partition we're currently yielded into
+      def stop_current_partition
+        stop_partition(
+          @current_message.topic,
+          @current_message.partition
+        )
+      end
+
+      # Stops processing of a given partition
+      # We expect the partition to be provided because of a scenario, where there is a
+      # multi-partition iteration and we want to stop a different partition that the one that
+      # is currently yielded.
+      #
+      # We pause it forever and no longer work with it.
+      #
+      # @param name [String] topic name of which partition we want to stop
+      # @param partition [Integer] partition we want to stop processing
+      def stop_partition(name, partition)
+        @stopped_partitions += 1
+
+        @current_consumer.pause(
+          Rdkafka::Consumer::TopicPartitionList.new(
+            name => [Partition.new(partition, 0)]
+          )
+        )
+      end
+
+      private
+
+      # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
+      def poll
+        @current_consumer.poll(@max_wait_time)
+      rescue Rdkafka::RdkafkaError => e
+        # End of partition
+        if e.code == :partition_eof
+          @stopped_partitions += 1
+
+          retry
+        end
+
+        raise e
+      end
+
+      # Converts raw rdkafka message into Karafka message
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [::Karafka::Messages::Message]
+      def build_message(message)
+        Messages::Builders::Message.call(
+          message,
+          @routing_topics.fetch(message.topic),
+          Time.now
+        )
+      end
+
+      # Do we have all the data we wanted or did every topic partition has reached eof.
+      # @return [Boolean]
+      def done?
+        @stopped_partitions >= @total_partitions
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Loader requires and loads all the pro components only when they are needed
+    class Loader
+      # There seems to be a conflict in between using two Zeitwerk instances and it makes lookups
+      # for nested namespaces instead of creating them.
+      # We require those not to deal with this and then all works as expected
+      FORCE_LOADED = %w[
+        active_job/dispatcher
+        processing/jobs/consume_non_blocking
+        processing/strategies/base
+        routing/features/base
+        encryption
+        encryption/cipher
+        encryption/setup/config
+        encryption/contracts/config
+        encryption/messages/parser
+      ].freeze
+
+      # Zeitwerk pro loader
+      # We need to have one per process, that's why it's set as a constant
+      PRO_LOADER = Zeitwerk::Loader.new
+
+      private_constant :PRO_LOADER
+
+      class << self
+        # Requires all the components without using them anywhere
+        def require_all
+          FORCE_LOADED.each { |file| require_relative(file) }
+
+          PRO_LOADER.push_dir(Karafka.core_root.join('pro'), namespace: Karafka::Pro)
+          PRO_LOADER.setup
+          PRO_LOADER.eager_load
+        end
+
+        # Loads all the pro components and configures them wherever it is expected
+        # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
+        #   components
+        def pre_setup_all(config)
+          features.each { |feature| feature.pre_setup(config) }
+
+          reconfigure(config)
+
+          load_topic_features
+        end
+
+        # Runs post setup features configuration operations
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        def post_setup_all(config)
+          features.each { |feature| feature.post_setup(config) }
+        end
+
+        private
+
+        # @return [Array<Module>] extra non-routing related pro features and routing components
+        #   that need to have some special configuration stuff injected into config, etc
+        def features
+          [
+            Encryption,
+            Cleaner
+          ]
+        end
+
+        # Sets proper config options to use pro components
+        # @param config [::Karafka::Core::Configurable::Node] root config node
+        def reconfigure(config)
+          icfg = config.internal
+
+          icfg.cli.contract = Contracts::ServerCliOptions.new
+
+          icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.partitioner_class = Processing::Partitioner
+          icfg.processing.scheduler = Processing::Scheduler.new
+          icfg.processing.jobs_builder = Processing::JobsBuilder.new
+          icfg.processing.strategy_selector = Processing::StrategySelector.new
+
+          icfg.active_job.consumer_class = ActiveJob::Consumer
+          icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
+          icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+
+          config.monitor.subscribe(PerformanceTracker.instance)
+        end
+
+        # Loads the Pro features of Karafka
+        # @note Object space lookup is not the fastest but we do it once during boot, so it's ok
+        def load_topic_features
+          ::Karafka::Pro::Routing::Features::Base.load_all
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/performance_tracker.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Tracker used to keep track of performance metrics
+    # It provides insights that can be used to optimize processing flow
+    class PerformanceTracker
+      include Singleton
+
+      # How many samples do we collect per topic partition
+      SAMPLES_COUNT = 200
+
+      private_constant :SAMPLES_COUNT
+
+      # Builds up nested concurrent hash for data tracking
+      def initialize
+        @processing_times = Concurrent::Map.new do |topics_hash, topic|
+          topics_hash.compute_if_absent(topic) do
+            Concurrent::Map.new do |partitions_hash, partition|
+              # This array does not have to be concurrent because we always access single
+              # partition data via instrumentation that operates in a single thread via consumer
+              partitions_hash.compute_if_absent(partition) { [] }
+            end
+          end
+        end
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @return [Float] p95 processing time of a single message from a single topic partition
+      def processing_time_p95(topic, partition)
+        values = @processing_times[topic][partition]
+
+        return 0 if values.empty?
+        return values.first if values.size == 1
+
+        percentile(0.95, values)
+      end
+
+      # @private
+      # @param event [Karafka::Core::Monitoring::Event] event details
+      # Tracks time taken to process a single message of a given topic partition
+      def on_consumer_consumed(event)
+        consumer = event[:caller]
+        messages = consumer.messages
+        topic = messages.metadata.topic
+        partition = messages.metadata.partition
+
+        samples = @processing_times[topic][partition]
+        samples << event[:time] / messages.count
+
+        return unless samples.size > SAMPLES_COUNT
+
+        samples.shift
+      end
+
+      private
+
+      # Computers the requested percentile out of provided values
+      # @param percentile [Float]
+      # @param values [Array<String>] all the values based on which we should
+      # @return [Float] computed percentile
+      def percentile(percentile, values)
+        values_sorted = values.sort
+
+        floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
+        mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+
+        values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
+      end
+    end
+  end
+end