karafka 2.0.0.rc1 → 2.0.0.rc4

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Dry::Events::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Dry::Events::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              metadata = event.payload[:caller].messages.metadata
+
+              extra_tags += [
+                "topic:#{metadata.topic}",
+                "partition:#{metadata.partition}"
+              ]
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Dry::Events::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Dry::Events::Event]
+          def on_consumer_consumed(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Dry::Events::Event]
+          def on_consumer_revoked(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Dry::Events::Event]
+          def on_consumer_shutdown(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Dry::Events::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Dry::Events::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -23,7 +23,9 @@ module Karafka
           dispatch_method: :produce_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with
           # each job.
-          partitioner: nil
+          partitioner: nil,
+          # Allows for usage of `:key` or `:partition_key`
+          partition_key_type: :key
         }.freeze
 
         private_constant :DEFAULTS
@@ -45,11 +47,12 @@ module Karafka
         # @return [Hash] hash with dispatch details to which we merge topic and payload
         def dispatch_details(job)
           partitioner = fetch_option(job, :partitioner, DEFAULTS)
+          key_type = fetch_option(job, :partition_key_type, DEFAULTS)
 
           return {} unless partitioner
 
           {
-            partition_key: partitioner.call(job)
+            key_type => partitioner.call(job)
           }
         end
       end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -14,13 +14,18 @@ module Karafka
     module ActiveJob
       # Contract for validating the options that can be altered with `#karafka_options` per job
       # class that works with Pro features.
-      class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract
-        # Dry types
-        Types = include Dry.Types()
-
-        params do
-          optional(:partitioner).value(Types.Interface(:call))
+      class JobOptionsContract < Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('job_options')
         end
+
+        optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+        optional(:partitioner) { |val| val.respond_to?(:call) }
+        optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
       end
     end
   end

data/lib/karafka/pro/contracts/base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Karafka Pro related contracts
+    module Contracts
+      # Base contract for Pro components contracts
+      class Base < ::Karafka::Contracts::Base
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/consumer_group.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correct Pro components setup on a consumer group and topic levels
+      class ConsumerGroup < Base
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:topics)
+
+          fetched_errors = []
+
+          data.fetch(:topics).each do |topic|
+            ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
+              fetched_errors << [[topic, key].flatten, value]
+            end
+          end
+
+          fetched_errors
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correct Pro components setup on a topic levels
+      class ConsumerGroupTopic < Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
+        end
+
+        virtual do |data|
+          next if data[:consumer] < Karafka::Pro::BaseConsumer
+
+          [[%i[consumer], :consumer_format]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -22,7 +22,11 @@ module Karafka
         processing/jobs_builder
         processing/coordinator
         processing/partitioner
-        routing/extensions
+        contracts/base
+        contracts/consumer_group
+        contracts/consumer_group_topic
+        routing/topic_extensions
+        routing/builder_extensions
         active_job/consumer
         active_job/dispatcher
         active_job/job_options_contract
@@ -37,6 +41,16 @@ module Karafka
         def setup(config)
           COMPONENTS.each { |component| require_relative(component) }
 
+          reconfigure(config)
+
+          load_routing_extensions
+        end
+
+        private
+
+        # Sets proper config options to use pro components
+        # @param config [WaterDrop::Configurable::Node] root config node
+        def reconfigure(config)
           icfg = config.internal
 
           icfg.processing.coordinator_class = Processing::Coordinator
@@ -48,10 +62,14 @@ module Karafka
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
           icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
-          ::Karafka::Routing::Topic.include(Routing::Extensions)
-
           config.monitor.subscribe(PerformanceTracker.instance)
         end
+
+        # Loads routing extensions
+        def load_routing_extensions
+          ::Karafka::Routing::Topic.include(Routing::TopicExtensions)
+          ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
+        end
       end
     end
   end

data/lib/karafka/pro/processing/partitioner.rb

@@ -27,9 +27,28 @@ module Karafka
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
           if ktopic.virtual_partitioner? && @concurrency > 1
-            messages
-              .group_by { |msg| ktopic.virtual_partitioner.call(msg).hash.abs % @concurrency }
-              .each { |group_id, messages_group| yield(group_id, messages_group) }
+            # We need to reduce it to number of threads, so the group_id is not a direct effect
+            # of the end user action. Otherwise the persistence layer for consumers would cache
+            # it forever and it would cause memory leaks
+            groupings = messages
+                        .group_by { |msg| ktopic.virtual_partitioner.call(msg) }
+                        .values
+
+            # Reduce the max concurrency to a size that matches the concurrency
+            # As mentioned above we cannot use the partitioning keys directly as it could cause
+            # memory leaks
+            #
+            # The algorithm here is simple, we assume that the most costly in terms of processing,
+            # will be processing of the biggest group and we reduce the smallest once to have
+            # max of groups equal to concurrency
+            while groupings.size > @concurrency
+              groupings.sort_by! { |grouping| -grouping.size }
+
+              # Offset order needs to be maintained for virtual partitions
+              groupings << (groupings.pop + groupings.pop).sort_by!(&:offset)
+            end
+
+            groupings.each_with_index { |messages_group, index| yield(index, messages_group) }
           else
             # When no virtual partitioner, works as regular one
             yield(0, messages)

data/lib/karafka/pro/routing/builder_extensions.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions for builder to be able to validate Pro components correct usage
+      module BuilderExtensions
+        # Validate consumer groups with pro contracts
+        # @param block [Proc] routing defining block
+        def draw(&block)
+          super
+
+          each do |consumer_group|
+            ::Karafka::Pro::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/{extensions.rb → topic_extensions.rb}

@@ -14,7 +14,7 @@ module Karafka
     # Pro routing components
     module Routing
       # Routing extensions that allow to configure some extra PRO routing options
-      module Extensions
+      module TopicExtensions
         class << self
           # @param base [Class] class we extend
           def included(base)

data/lib/karafka/processing/jobs_queue.rb

@@ -119,6 +119,17 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
+      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
+      #
+      # @return [Hash] hash with basic usage statistics of this queue.
+      def statistics
+        {
+          processing: size - @queue.size,
+          enqueued: @queue.size
+        }.freeze
+      end
+
       private
 
       # @param group_id [String] id of the group in which jobs we're interested.

data/lib/karafka/processing/worker.rb

@@ -47,9 +47,11 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
-          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
             job.before_call
 
             # If a job is marked as non blocking, we can run a tick in the job queue and if there

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend Dry::Configurable
+      extend ::Karafka::Core::Configurable
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
@@ -60,9 +60,9 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 1_000
+      setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
-      setting :max_wait_time, default: 10_000
+      setting :max_wait_time, default: 1_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
       #   longer waits for the consumers to stop gracefully but instead we force terminate
       #   everything.
@@ -123,6 +123,10 @@ module Karafka
         end
       end
 
+      # This will load all the defaults that can be later overwritten.
+      # Thanks to that we have an initial state out of the box.
+      configure
+
       class << self
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance

data/lib/karafka/templates/example_consumer.rb.erb

@@ -7,10 +7,10 @@ class ExampleConsumer < ApplicationConsumer
   end
 
   # Run anything upon partition being revoked
-  # def on_revoked
+  # def revoked
   # end
 
   # Define here any teardown things you want when Karafka server stops
-  # def on_shutdown
+  # def shutdown
   # end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.rc1'
+  VERSION = '2.0.0.rc4'
 end

data/lib/karafka.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 %w[
+  karafka-core
   delegate
   English
   rdkafka
@@ -12,10 +13,6 @@
   openssl
   base64
   date
-  dry-configurable
-  dry-validation
-  dry/events/publisher
-  dry/monitor/notifications
   zeitwerk
 ].each(&method(:require))
 
@@ -90,5 +87,7 @@ loader = Zeitwerk::Loader.for_gem
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
 # Do not load pro components, this will be handled by license manager
 loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
 loader.setup
 loader.eager_load