karafka 2.0.0 → 2.1.0

data/config/locales/pro_errors.yml

@@ -0,0 +1,34 @@
+en:
+  validations:
+    topic:
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.max_partitions_format: needs to be equal or more than 1
+
+      long_running_job.active_format: needs to be either true or false
+
+      dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
+
+      throttling.active_format: needs to be either true or false
+      throttling.limit_format: needs to be equal or more than 1
+      throttling.interval_format: needs to be equal or more than 1
+
+      filtering.active_missing: needs to be present
+      filtering.factory_format: 'needs to respond to #call'
+      filtering.factories_format: 'needs to contain only factories responding to #call'
+      filtering.active_format: 'needs to be boolean'
+
+      expiring.ttl_format: 'needs to be equal or more than 0 and an integer'
+      expiring.active_format: 'needs to be boolean'
+
+      delaying.delay_format: 'needs to be equal or more than 0 and an integer'
+      delaying.active_format: 'needs to be boolean'
+
+    config:
+      encryption.active_format: 'needs to be either true or false'
+      encryption.public_key_invalid: 'is not a valid public RSA key'
+      encryption.public_key_needs_to_be_public: 'is a private RSA key not a public one'
+      encryption.private_keys_format: 'needs to be a hash of version and private key value'
+      encryption.private_keys_need_to_be_private: 'all keys need to be private'
+      encryption.version_format: must be a non-empty string
+      encryption.public_key_format: 'is not a valid public RSA key'
+      encryption.private_keys_invalid: 'contains an invalid private RSA key string'

data/docker-compose.yml

@@ -3,8 +3,10 @@ services:
   zookeeper:
     container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
+    restart: on-failure
     ports:
       - '2181:2181'
+
   kafka:
     container_name: karafka_20_kafka
     image: wurstmeister/kafka
@@ -16,38 +18,10 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_00_02:2:1,\
-         integrations_01_02:2:1,\
-         integrations_02_02:2:1,\
-         integrations_03_02:2:1,\
-         integrations_04_02:2:1,\
-         integrations_05_02:2:1,\
-         integrations_06_02:2:1,\
-         integrations_07_02:2:1,\
-         integrations_08_02:2:1,\
-         integrations_09_02:2:1,\
-         integrations_10_02:2:1,\
-         integrations_11_02:2:1,\
-         integrations_12_02:2:1,\
-         integrations_13_02:2:1,\
-         integrations_14_02:2:1,\
-         integrations_15_02:2:1,\
-         integrations_16_02:2:1,\
-         integrations_17_02:2:1,\
-         integrations_18_02:2:1,\
-         integrations_19_02:2:1,\
-         integrations_20_02:2:1,\
-         integrations_21_02:2:1,\
-         integrations_00_03:3:1,\
-         integrations_01_03:3:1,\
-         integrations_02_03:3:1,\
-         integrations_03_03:3:1,\
-         integrations_04_03:3:1,\
-         integrations_00_10:10:1,\
-         integrations_01_10:10:1,\
-         benchmarks_00_01:1:1,\
+        "benchmarks_00_01:1:1,\
          benchmarks_00_05:5:1,\
          benchmarks_01_05:5:1,\
          benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure

data/karafka.gemspec

@@ -10,31 +10,38 @@ Gem::Specification.new do |spec|
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld']
-  spec.email       = %w[maciej@mensfeld.pl]
+  spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails'
-  spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.licenses    = ['LGPL-3.0', 'Commercial']
+  spec.licenses    = %w[LGPL-3.0 Commercial]
+  spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
+  spec.description = <<-DESC
+    Karafka is Ruby and Rails efficient Kafka processing framework.
 
-  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
-  spec.add_dependency 'rdkafka', '>= 0.12'
+    Karafka allows you to capture everything that happens in your systems in large scale,
+    without having to focus on things that are not your business domain.
+  DESC
+
+  spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.7.0'
-
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  spec.cert_chain    = %w[certs/mensfeld.pem]
+  spec.cert_chain    = %w[certs/cert_chain.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
 
   spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/lib/active_job/karafka.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 begin
-  require 'active_job'
+  # Do not load active job if already loaded
+  require 'active_job' unless Object.const_defined?('ActiveJob')
+
   require_relative 'queue_adapters/karafka_adapter'
 
   module ActiveJob
@@ -10,12 +12,6 @@ begin
     module Karafka
     end
   end
-
-  # We extend routing builder by adding a simple wrapper for easier jobs topics defining
-  # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
-  # the railtie initializer would be too late
-  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::Routing::Extensions
-  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::Routing::Extensions
 rescue LoadError
   # We extend ActiveJob stuff in the railtie
 end

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -11,7 +11,13 @@ module ActiveJob
       #
       # @param job [Object] job that should be enqueued
       def enqueue(job)
-        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
+        ::Karafka::App.config.internal.active_job.dispatcher.dispatch(job)
+      end
+
+      # Enqueues multiple jobs in one go
+      # @param jobs [Array<Object>] jobs that we want to enqueue
+      def enqueue_all(jobs)
+        ::Karafka::App.config.internal.active_job.dispatcher.dispatch_many(jobs)
       end
 
       # Raises info, that Karafka backend does not support scheduling jobs

data/lib/karafka/active_job/consumer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for all the ActiveJob related things from within Karafka
   module ActiveJob
     # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
     # It marks the offset after each message, so we make sure, none of the jobs is executed twice
@@ -11,16 +12,38 @@ module Karafka
         messages.each do |message|
           break if Karafka::App.stopping?
 
-          ::ActiveJob::Base.execute(
-            # We technically speaking could set this as deserializer and reference it from the
-            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
-            # the ActiveJob setup here
-            ::ActiveSupport::JSON.decode(message.raw_payload)
-          )
+          consume_job(message)
 
           mark_as_consumed(message)
         end
       end
+
+      private
+
+      # Consumes a message with the job and runs needed instrumentation
+      #
+      # @param job_message [Karafka::Messages::Message] message with active job
+      def consume_job(job_message)
+        with_deserialized_job(job_message) do |job|
+          tags.add(:job_class, job['job_class'])
+
+          payload = { caller: self, job: job, message: job_message }
+
+          # We publish both to make it consistent with `consumer.x` events
+          Karafka.monitor.instrument('active_job.consume', payload)
+          Karafka.monitor.instrument('active_job.consumed', payload) do
+            ::ActiveJob::Base.execute(job)
+          end
+        end
+      end
+
+      # @param job_message [Karafka::Messages::Message] message with active job
+      def with_deserialized_job(job_message)
+        # We technically speaking could set this as deserializer and reference it from the
+        # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+        # the ActiveJob setup here
+        yield ::ActiveSupport::JSON.decode(job_message.raw_payload)
+      end
     end
   end
 end

data/lib/karafka/active_job/current_attributes/loading.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Module expanding the job deserialization to extract current attributes and load them
+      # for the time of the job execution
+      module Loading
+        # @param job_message [Karafka::Messages::Message] message with active job
+        def with_deserialized_job(job_message)
+          super(job_message) do |job|
+            resetable = []
+
+            _cattr_klasses.each do |key, cattr_klass_str|
+              next unless job.key?(key)
+
+              attributes = job.delete(key)
+
+              cattr_klass = cattr_klass_str.constantize
+
+              attributes.each do |name, value|
+                cattr_klass.public_send("#{name}=", value)
+              end
+
+              resetable << cattr_klass
+            end
+
+            yield(job)
+
+            resetable.each(&:reset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes/persistence.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Module adding the current attributes persistence into the ActiveJob jobs
+      module Persistence
+        # Alters the job serialization to inject the current attributes into the json before we
+        # send it to Kafka
+        #
+        # @param job [ActiveJob::Base] job
+        def serialize_job(job)
+          json = super(job)
+
+          _cattr_klasses.each do |key, cattr_klass_str|
+            next if json.key?(key)
+
+            attrs = cattr_klass_str.constantize.attributes
+
+            json[key] = attrs unless attrs.empty?
+          end
+
+          json
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'active_support/current_attributes'
+require_relative 'current_attributes/loading'
+require_relative 'current_attributes/persistence'
+
+# This code is based on Sidekiqs approach to persisting current attributes
+# @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
+module Karafka
+  module ActiveJob
+    # Module that allows to persist current attributes on Karafka jobs
+    module CurrentAttributes
+      # Allows for persistence of given current attributes via AJ + Karafka
+      #
+      # @param klasses [Array<String, Class>] classes or names of the current attributes classes
+      def persist(*klasses)
+        # Support for providing multiple classes
+        klasses = Array(klasses).flatten
+
+        [Dispatcher, Consumer]
+          .reject { |expandable| expandable.respond_to?(:_cattr_klasses) }
+          .each { |expandable| expandable.class_attribute :_cattr_klasses, default: {} }
+
+        # Do not double inject in case of running persist multiple times
+        Dispatcher.prepend(Persistence) unless Dispatcher.ancestors.include?(Persistence)
+        Consumer.prepend(Loading) unless Consumer.ancestors.include?(Loading)
+
+        klasses.map(&:to_s).each do |stringified_klass|
+          # Prevent registering same klass multiple times
+          next if Dispatcher._cattr_klasses.value?(stringified_klass)
+
+          key = "cattr_#{Dispatcher._cattr_klasses.count}"
+
+          Dispatcher._cattr_klasses[key] = stringified_klass
+          Consumer._cattr_klasses[key] = stringified_klass
+        end
+      end
+
+      module_function :persist
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -7,20 +7,45 @@ module Karafka
       # Defaults for dispatching
       # The can be updated by using `#karafka_options` on the job
       DEFAULTS = {
-        dispatch_method: :produce_async
+        dispatch_method: :produce_async,
+        dispatch_many_method: :produce_many_async
       }.freeze
 
       private_constant :DEFAULTS
 
       # @param job [ActiveJob::Base] job
-      def call(job)
+      def dispatch(job)
         ::Karafka.producer.public_send(
           fetch_option(job, :dispatch_method, DEFAULTS),
           topic: job.queue_name,
-          payload: ::ActiveSupport::JSON.encode(job.serialize)
+          payload: ::ActiveSupport::JSON.encode(serialize_job(job))
         )
       end
 
+      # Bulk dispatches multiple jobs using the Rails 7.1+ API
+      # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+      def dispatch_many(jobs)
+        # Group jobs by their desired dispatch method
+        # It can be configured per job class, so we need to make sure we divide them
+        dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+        jobs.each do |job|
+          d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+          dispatches[d_method] << {
+            topic: job.queue_name,
+            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+          }
+        end
+
+        dispatches.each do |type, messages|
+          ::Karafka.producer.public_send(
+            type,
+            messages
+          )
+        end
+      end
+
       private
 
       # @param job [ActiveJob::Base] job
@@ -33,6 +58,12 @@ module Karafka
           .karafka_options
           .fetch(key, defaults.fetch(key))
       end
+
+      # @param job [ActiveJob::Base] job
+      # @return [Hash] json representation of the job
+      def serialize_job(job)
+        job.serialize
+      end
     end
   end
 end

data/lib/karafka/active_job/job_options_contract.rb

@@ -10,12 +10,23 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('job_options')
       end
 
-      optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+      optional(:dispatch_method) do |val|
+        %i[
+          produce_async
+          produce_sync
+        ].include?(val)
+      end
+      optional(:dispatch_many_method) do |val|
+        %i[
+          produce_many_async
+          produce_many_sync
+        ].include?(val)
+      end
     end
   end
 end

data/lib/karafka/admin.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  module Admin
+    # A fake admin topic representation that we use for messages fetched using this API
+    # We cannot use the topics directly because we may want to request data from topics that we
+    # do not have in the routing
+    Topic = Struct.new(:name, :deserializer)
+
+    # We wait only for this amount of time before raising error as we intercept this error and
+    # retry after checking that the operation was finished or failed using external factor.
+    MAX_WAIT_TIMEOUT = 1
+
+    # How many times should be try. 1 x 60 => 60 seconds wait in total
+    MAX_ATTEMPTS = 60
+
+    # Defaults for config
+    CONFIG_DEFAULTS = {
+      'group.id': 'karafka_admin',
+      # We want to know when there is no more data not to end up with an endless loop
+      'enable.partition.eof': true,
+      'statistics.interval.ms': 0,
+      # Fetch at most 5 MBs when using admin
+      'fetch.message.max.bytes': 5 * 1_048_576,
+      # Do not commit offset automatically, this prevents offset tracking for operations involving
+      # a consumer instance
+      'enable.auto.commit': false
+    }.freeze
+
+    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
+
+    class << self
+      # Allows us to read messages from the topic
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @param count [Integer] how many messages we want to get at most
+      # @param start_offset [Integer] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset
+      # @param settings [Hash] kafka extra settings (optional)
+      #
+      # @return [Array<Karafka::Messages::Message>] array with messages
+      def read_topic(name, partition, count, start_offset = -1, settings = {})
+        messages = []
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        low_offset, high_offset = nil
+
+        with_consumer(settings) do |consumer|
+          low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
+
+          # Select offset dynamically if -1 or less
+          start_offset = high_offset - count if start_offset.negative?
+
+          # Build the requested range - since first element is on the start offset we need to
+          # subtract one from requested count to end up with expected number of elements
+          requested_range = (start_offset..start_offset + (count - 1))
+          # Establish theoretical available range. Note, that this does not handle cases related to
+          # log retention or compaction
+          available_range = (low_offset..high_offset)
+          # Select only offset that we can select. This will remove all the potential offsets that
+          # are below the low watermark offset
+          possible_range = requested_range.select { |offset| available_range.include?(offset) }
+
+          start_offset = possible_range.first
+          count = possible_range.count
+
+          tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
+          consumer.assign(tpl)
+
+          # We should poll as long as we don't have all the messages that we need or as long as
+          # we do not read all the messages from the topic
+          loop do
+            # If we've got as many messages as we've wanted stop
+            break if messages.size >= count
+
+            message = consumer.poll(200)
+
+            next unless message
+
+            # If the message we've got is beyond the requested range, stop
+            break unless possible_range.include?(message.offset)
+
+            messages << message
+          rescue Rdkafka::RdkafkaError => e
+            # End of partition
+            break if e.code == :partition_eof
+
+            raise e
+          end
+        end
+
+        # Use topic from routes if we can match it or create a dummy one
+        # Dummy one is used in case we cannot match the topic with routes. This can happen
+        # when admin API is used to read topics that are not part of the routing
+        topic = ::Karafka::Routing::Router.find_or_initialize_by_name(name)
+
+        messages.map! do |message|
+          Messages::Builders::Message.call(
+            message,
+            topic,
+            Time.now
+          )
+        end
+      end
+
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          handler = admin.create_topic(name, partitions, replication_factor, topic_config)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          handler = admin.delete_topic(name)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { !topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Creates more partitions for a given topic
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] total number of partitions we expect to end up with
+      def create_partitions(name, partitions)
+        with_admin do |admin|
+          handler = admin.create_partitions(name, partitions)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topic(name).fetch(:partition_count) >= partitions }
+          )
+        end
+      end
+
+      # Fetches the watermark offsets for a given topic partition
+      #
+      # @param name [String, Symbol] topic name
+      # @param partition [Integer] partition
+      # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
+      def read_watermark_offsets(name, partition)
+        with_consumer do |consumer|
+          consumer.query_watermark_offsets(name, partition)
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+        end
+      end
+
+      # Creates consumer instance and yields it. After usage it closes the consumer instance
+      # This API can be used in other pieces of code and allows for low-level consumer usage
+      #
+      # @param settings [Hash] extra settings to customize consumer
+      def with_consumer(settings = {})
+        consumer = config(:consumer, settings).consumer
+        yield(consumer)
+      ensure
+        consumer&.close
+      end
+
+      private
+
+      # @return [Array<String>] topics names
+      def topics_names
+        cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
+      end
+
+      # Finds details about given topic
+      # @param name [String] topic name
+      # @return [Hash] topic details
+      def topic(name)
+        cluster_info.topics.find { |topic| topic[:topic_name] == name }
+      end
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = config(:producer, {}).admin
+        yield(admin)
+      ensure
+        admin&.close
+      end
+
+      # There are some cases where rdkafka admin operations finish successfully but without the
+      # callback being triggered to materialize the post-promise object. Until this is fixed we
+      # can figure out, that operation we wanted to do finished successfully by checking that the
+      # effect of the command (new topic, more partitions, etc) is handled. Exactly for that we
+      # use the breaker. It we get a timeout, we can check that what we wanted to achieve has
+      # happened via the breaker check, hence we do not need to wait any longer.
+      #
+      # @param handler [Proc] the wait handler operation
+      # @param breaker [Proc] extra condition upon timeout that indicates things were finished ok
+      def with_re_wait(handler, breaker)
+        attempt ||= 0
+        attempt += 1
+
+        handler.call
+      rescue Rdkafka::AbstractHandle::WaitTimeoutError
+        return if breaker.call
+
+        retry if attempt <= MAX_ATTEMPTS
+
+        raise
+      end
+
+      # @param type [Symbol] type of config we want
+      # @param settings [Hash] extra settings for config (if needed)
+      # @return [::Rdkafka::Config] rdkafka config
+      def config(type, settings)
+        config_hash = Karafka::Setup::AttributesMap.public_send(
+          type,
+          Karafka::App.config.kafka.dup.merge(CONFIG_DEFAULTS).merge!(settings)
+        )
+
+        ::Rdkafka::Config.new(config_hash)
+      end
+    end
+  end
+end