karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related messages components
+      module Messages
+        # Middleware for WaterDrop. It automatically encrypts messages payload.
+        # It is injected only if encryption is enabled.
+        class Middleware
+          # @param message [Hash] WaterDrop message hash
+          # @return [Hash] hash with encrypted payload and encryption version indicator
+          def call(message)
+            message[:headers] ||= {}
+            message[:headers]['encryption'] = version
+            message[:payload] = cipher.encrypt(message[:payload])
+            message
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [String] encryption version
+          def version
+            @version ||= ::Karafka::App.config.encryption.version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      module Messages
+        # Pro parser that takes into consideration encryption usage
+        # @note There may be a case where someone decides not to encrypt data and we start getting
+        #   unencrypted payloads. That is why we always rely on message headers for encryption
+        #   indication.
+        class Parser < ::Karafka::Messages::Parser
+          # @param message [::Karafka::Messages::Message]
+          # @return [Object] deserialized payload
+          def call(message)
+            if active? && message.headers.key?('encryption')
+              # Decrypt raw payload so it can be handled by the default parser logic
+              message.raw_payload = cipher.decrypt(
+                message.headers['encryption'],
+                message.raw_payload
+              )
+            end
+
+            super(message)
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [Boolean] is encryption active
+          def active?
+            return @active unless @active.nil?
+
+            @active = ::Karafka::App.config.encryption.active
+
+            @active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/setup/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Setup and config related encryption components
+      module Setup
+        # Config for encryption
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          # Should this feature be in use
+          setting(:active, default: false)
+
+          # Supporting versions allows us to be able to rotate private and public keys in case
+          # we would need this. We can increase the version, rotate and Karafka when decrypting
+          # will figure out proper private key based on the version
+          setting(:version, default: '1')
+
+          # We always support one public key for producing messages
+          # Public key needs to be always present even if we do not plan to produce messages from
+          # a Karafka process. This is because of the web-ui and potentially other cases like this
+          setting(:public_key, default: '')
+
+          # Private keys in pem format, where the key is the version and value is the key.
+          # This allows us to support key rotation
+          setting(:private_keys, default: {})
+
+          # Cipher used to encrypt and decrypt data
+          setting(:cipher, default: Encryption::Cipher.new)
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Out of the box encryption engine for both Karafka and WaterDrop
+    # It uses asymmetric encryption via RSA. We use asymmetric so we can have producers that won't
+    # have ability (when private key not added) to decrypt messages.
+    module Encryption
+      class << self
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:encryption, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          Encryption::Contracts::Config.new.validate!(config.to_h)
+
+          # Don't inject extra components if encryption is not active
+          return unless config.encryption.active
+
+          # This parser is encryption aware
+          config.internal.messages.parser = Messages::Parser.new
+
+          # Encryption for WaterDrop
+          config.producer.middleware.append(Messages::Middleware.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Topic iterator allows you to iterate over topic/partition data and perform lookups for
+    # information that you need.
+    #
+    # It supports early stops on finding the requested data and allows for seeking till
+    # the end. It also allows for signaling, when a given message should be last out of certain
+    # partition, but we still want to continue iterating in other messages.
+    #
+    # It does **not** create a consumer group and does not have any offset management.
+    class Iterator
+      # Local partition reference for librdkafka
+      Partition = Struct.new(:partition, :offset)
+
+      private_constant :Partition
+
+      # A simple API allowing to iterate over topic/partition data, without having to subscribe
+      # and deal with rebalances. This API allows for multi-partition streaming and is optimized
+      # for data lookups. It allows for explicit stopping iteration over any partition during
+      # the iteration process, allowing for optimized lookups.
+      #
+      # @param topics [Array<String>, Hash] list of strings if we want to subscribe to multiple
+      #   topics and all of their partitions or a hash where keys are the topics and values are
+      #   hashes with partitions and their initial offsets.
+      # @param settings [Hash] extra settings for the consumer. Please keep in mind, that if
+      #   overwritten, you may want to include `auto.offset.reset` to match your case.
+      # @param yield_nil [Boolean] should we yield also `nil` values when poll returns nothing.
+      #   Useful in particular for long-living iterators.
+      #
+      # @note It is worth keeping in mind, that this API also needs to operate within
+      #   `max.poll.interval.ms` limitations on each iteration
+      #
+      # @note In case of a never-ending iterator, you need to set `enable.partition.eof` to `false`
+      #   so we don't stop polling data even when reaching the end (end on a given moment)
+      def initialize(
+        topics,
+        settings: { 'auto.offset.reset': 'beginning' },
+        yield_nil: false
+      )
+        @topics_with_partitions = expand_topics_with_partitions(topics)
+
+        @routing_topics = @topics_with_partitions.map do |name, _|
+          [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
+        end.to_h
+
+        @total_partitions = @topics_with_partitions.map(&:last).sum(&:count)
+
+        @stopped_partitions = 0
+
+        @settings = settings
+        @yield_nil = yield_nil
+      end
+
+      # Iterates over requested topic partitions and yields the results with the iterator itself
+      # Iterator instance is yielded because one can run `stop_partition` to stop iterating over
+      # part of data. It is useful for scenarios where we are looking for some information in all
+      # the partitions but once we found it, given partition data is no longer needed and would
+      # only eat up resources.
+      def each
+        Admin.with_consumer(@settings) do |consumer|
+          tpl = tpl_with_expanded_offsets(consumer)
+          consumer.assign(tpl)
+
+          # We need this for self-referenced APIs like pausing
+          @current_consumer = consumer
+
+          # Stream data until we reach the end of all the partitions or until the end user
+          # indicates that they are done
+          until done?
+            message = poll(200)
+
+            # Skip nils if not explicitly required
+            next if message.nil? && !@yield_nil
+
+            if message
+              @current_message = build_message(message)
+
+              yield(@current_message, self)
+            else
+              yield(nil, self)
+            end
+          end
+
+          @current_message = nil
+          @current_consumer = nil
+        end
+
+        # Reset so we can use the same iterator again if needed
+        @stopped_partitions = 0
+      end
+
+      # Stops the partition we're currently yielded into
+      def stop_current_partition
+        stop_partition(
+          @current_message.topic,
+          @current_message.partition
+        )
+      end
+
+      # Stops processing of a given partition
+      # We expect the partition to be provided because of a scenario, where there is a
+      # multi-partition iteration and we want to stop a different partition that the one that
+      # is currently yielded.
+      #
+      # We pause it forever and no longer work with it.
+      #
+      # @param name [String] topic name of which partition we want to stop
+      # @param partition [Integer] partition we want to stop processing
+      def stop_partition(name, partition)
+        @stopped_partitions += 1
+
+        @current_consumer.pause(
+          Rdkafka::Consumer::TopicPartitionList.new(
+            name => [Partition.new(partition, 0)]
+          )
+        )
+      end
+
+      private
+
+      # Expands topics to which we want to subscribe with partitions information in case this
+      # info is not provided. For our convenience we want to support 5 formats of defining
+      # the subscribed topics:
+      #
+      # - 'topic1' - just a string with one topic name
+      # - ['topic1', 'topic2'] - just the names
+      # - { 'topic1' => -100 } - names with negative lookup offset
+      # - { 'topic1' => { 0 => 5 } } - names with exact partitions offsets
+      # - { 'topic1' => { 0 => -5 }, 'topic2' => { 1 => 5 } } - with per partition negative offsets
+      #
+      # @param topics [Array, Hash] topics definitions
+      # @return [Hash] hash with topics containing partitions definitions
+      def expand_topics_with_partitions(topics)
+        # Simplification for the single topic case
+        topics = [topics] if topics.is_a?(String)
+        # If we've got just array with topics, we need to convert that into a representation
+        # that we can expand with offsets
+        topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
+
+        expanded = Hash.new { |h, k| h[k] = {} }
+
+        topics.map do |topic, details|
+          if details.is_a?(Hash)
+            details.each do |partition, offset|
+              expanded[topic][partition] = offset
+            end
+          else
+            partition_count(topic.to_s).times do |partition|
+              # If no offsets are provided, we just start from zero
+              expanded[topic][partition] = details || 0
+            end
+          end
+        end
+
+        expanded
+      end
+
+      # @param timeout [Integer] timeout in ms
+      # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
+      def poll(timeout)
+        @current_consumer.poll(timeout)
+      rescue Rdkafka::RdkafkaError => e
+        # End of partition
+        if e.code == :partition_eof
+          @stopped_partitions += 1
+
+          retry
+        end
+
+        raise e
+      end
+
+      # Converts raw rdkafka message into Karafka message
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [::Karafka::Messages::Message]
+      def build_message(message)
+        Messages::Builders::Message.call(
+          message,
+          @routing_topics.fetch(message.topic),
+          Time.now
+        )
+      end
+
+      # Do we have all the data we wanted or did every topic partition has reached eof.
+      # @return [Boolean]
+      def done?
+        @stopped_partitions >= @total_partitions
+      end
+
+      # Builds the tpl representing all the subscriptions we want to run
+      #
+      # Additionally for negative offsets, does the watermark calculation where to start
+      #
+      # @param consumer [Rdkafka::Consumer] consumer we need in case of negative offsets as
+      #   negative are going to be used to do "give me last X". We use the already initialized
+      #   consumer instance, not to start another one again.
+      # @return [Rdkafka::Consumer::TopicPartitionList]
+      def tpl_with_expanded_offsets(consumer)
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+        @topics_with_partitions.each do |name, partitions|
+          partitions_with_offsets = {}
+
+          # When no offsets defined, we just start from zero
+          if partitions.is_a?(Array) || partitions.is_a?(Range)
+            partitions_with_offsets = partitions.map { |partition| [partition, 0] }.to_h
+          else
+            # When offsets defined, we can either use them if positive or expand and move back
+            # in case of negative (-1000 means last 1000 messages, etc)
+            partitions.each do |partition, offset|
+              if offset.negative?
+                _, high_watermark_offset = consumer.query_watermark_offsets(name, partition)
+                # We add because this offset is negative
+                partitions_with_offsets[partition] = high_watermark_offset + offset
+              else
+                partitions_with_offsets[partition] = offset
+              end
+            end
+          end
+
+          tpl.add_topic_and_partitions_with_offsets(name, partitions_with_offsets)
+        end
+
+        tpl
+      end
+
+      # @param name [String] topic name
+      # @return [Integer] number of partitions of the topic we want to iterate over
+      def partition_count(name)
+        Admin
+          .cluster_info
+          .topics
+          .find { |topic| topic.fetch(:topic_name) == name }
+          .fetch(:partitions)
+          .count
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -13,43 +15,66 @@ module Karafka
   module Pro
     # Loader requires and loads all the pro components only when they are needed
     class Loader
-      # All the pro components that need to be loaded
-      COMPONENTS = %w[
-        base_consumer
-        performance_tracker
-        processing/scheduler
-        processing/jobs/consume_non_blocking
-        processing/jobs_builder
-        processing/coordinator
-        processing/partitioner
-        contracts/base
-        contracts/consumer_group
-        contracts/consumer_group_topic
-        routing/topic_extensions
-        routing/builder_extensions
-        active_job/consumer
+      # There seems to be a conflict in between using two Zeitwerk instances and it makes lookups
+      # for nested namespaces instead of creating them.
+      # We require those not to deal with this and then all works as expected
+      FORCE_LOADED = %w[
         active_job/dispatcher
-        active_job/job_options_contract
+        processing/jobs/consume_non_blocking
+        processing/strategies/base
+        routing/features/base
+        encryption
+        encryption/cipher
+        encryption/setup/config
+        encryption/contracts/config
+        encryption/messages/parser
       ].freeze
 
-      private_constant :COMPONENTS
+      # Zeitwerk pro loader
+      # We need to have one per process, that's why it's set as a constant
+      PRO_LOADER = Zeitwerk::Loader.new
+
+      private_constant :PRO_LOADER
 
       class << self
+        # Requires all the components without using them anywhere
+        def require_all
+          FORCE_LOADED.each { |file| require_relative(file) }
+
+          PRO_LOADER.push_dir(Karafka.core_root.join('pro'), namespace: Karafka::Pro)
+          PRO_LOADER.setup
+          PRO_LOADER.eager_load
+        end
+
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def setup(config)
-          COMPONENTS.each { |component| require_relative(component) }
+        def pre_setup(config)
+          features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
 
-          load_routing_extensions
+          load_topic_features
+        end
+
+        # Runs post setup features configuration operations
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        def post_setup(config)
+          features.each { |feature| feature.post_setup(config) }
         end
 
         private
 
+        # @return [Array<Module>] extra non-routing related pro features
+        def features
+          [
+            Encryption
+          ]
+        end
+
         # Sets proper config options to use pro components
-        # @param config [WaterDrop::Configurable::Node] root config node
+        # @param config [::Karafka::Core::Configurable::Node] root config node
         def reconfigure(config)
           icfg = config.internal
 
@@ -57,6 +82,7 @@ module Karafka
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler = Processing::Scheduler.new
           icfg.processing.jobs_builder = Processing::JobsBuilder.new
+          icfg.processing.strategy_selector = Processing::StrategySelector.new
 
           icfg.active_job.consumer_class = ActiveJob::Consumer
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
@@ -65,10 +91,10 @@ module Karafka
           config.monitor.subscribe(PerformanceTracker.instance)
         end
 
-        # Loads routing extensions
-        def load_routing_extensions
-          ::Karafka::Routing::Topic.include(Routing::TopicExtensions)
-          ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
+        # Loads the Pro features of Karafka
+        # @note Object space lookup is not the fastest but we do it once during boot, so it's ok
+        def load_topic_features
+          ::Karafka::Pro::Routing::Features::Base.load_all
         end
       end
     end

data/lib/karafka/pro/performance_tracker.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -23,11 +25,13 @@ module Karafka
 
       # Builds up nested concurrent hash for data tracking
       def initialize
-        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
-          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
-            # This array does not have to be concurrent because we always access single partition
-            # data via instrumentation that operates in a single thread via consumer
-            partitions_hash[partition] = []
+        @processing_times = Concurrent::Map.new do |topics_hash, topic|
+          topics_hash.compute_if_absent(topic) do
+            Concurrent::Map.new do |partitions_hash, partition|
+              # This array does not have to be concurrent because we always access single
+              # partition data via instrumentation that operates in a single thread via consumer
+              partitions_hash.compute_if_absent(partition) { [] }
+            end
           end
         end
       end

data/lib/karafka/pro/processing/collapser.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manages the collapse of virtual partitions
+      # Since any non-virtual partition is actually a virtual partition of size one, we can use
+      # it in a generic manner without having to distinguish between those cases.
+      #
+      # We need to have notion of the offset until we want to collapse because upon pause and retry
+      # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
+      # (different) dataset and without tracking the end of collapse, there would be a chance for
+      # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
+      # messages from the corrupted batch are processed.
+      class Collapser
+        # When initialized, nothing is collapsed
+        def initialize
+          @collapsed = false
+          @until_offset = -1
+          @mutex = Mutex.new
+        end
+
+        # @return [Boolean] Should we collapse into a single consumer
+        def collapsed?
+          @collapsed
+        end
+
+        # Collapse until given offset. Until given offset is encountered or offset bigger than that
+        # we keep collapsing.
+        # @param offset [Integer] offset until which we keep the collapse
+        def collapse_until!(offset)
+          @mutex.synchronize do
+            # We check it here in case after a pause and re-fetch we would get less messages and
+            # one of them would cause an error. We do not want to overwrite the offset here unless
+            # it is bigger.
+            @until_offset = offset if offset > @until_offset
+          end
+        end
+
+        # Sets the collapse state based on the first collective offset that we are going to process
+        # and makes the decision whether or not we need to still keep the collapse.
+        # @param first_offset [Integer] first offset from a collective batch
+        def refresh!(first_offset)
+          @mutex.synchronize do
+            @collapsed = first_offset < @until_offset
+          end
+        end
+      end
+    end
+  end
+end