karafka 2.0.0 → 2.1.0

data/lib/karafka/contracts/server_cli_options.rb

@@ -7,25 +7,85 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
+      %i[
+        include
+        exclude
+      ].each do |action|
+        optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+        optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+        optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
 
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless data.key?(:consumer_groups)
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_consumer_groups")
+
+          # If there were no consumer_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+          next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+          # Found unknown consumer groups
+          [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_subscription_groups")
+
+          # If there were no subscription_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          subscription_groups = Karafka::App
+                                .consumer_groups
+                                .map(&:subscription_groups)
+                                .flatten
+                                .map(&:name)
+
+          next if (value - subscription_groups).empty?
 
-        value = data.fetch(:consumer_groups)
+          # Found unknown subscription groups
+          [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_topics")
+
+          # If there were no topics declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          topics = Karafka::App
+                   .consumer_groups
+                   .map(&:subscription_groups)
+                   .flatten
+                   .map(&:topics)
+                   .map { |gtopics| gtopics.map(&:name) }
+                   .flatten
+
+          next if (value - topics).empty?
+
+          # Found unknown topics
+          [[[:"#{action}_topics"], :topics_inclusion]]
+        end
+      end
+
+      # Makes sure we have anything to subscribe to when we start the server
+      virtual do |_, errors|
+        next unless errors.empty?
 
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        next if value.nil?
-        next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
+        next unless Karafka::App.subscription_groups.empty?
 
-        [[%i[consumer_groups], :consumer_groups_inclusion]]
+        [[%i[include_topics], :topics_missing]]
       end
     end
   end

data/lib/karafka/contracts/topic.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Consumer group topic validation rules.
+    class Topic < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('topic')
+      end
+
+      required(:deserializer) { |val| !val.nil? }
+      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:initial_offset) { |val| %w[earliest latest].include?(val) }
+      required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
+      required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:active) { |val| [true, false].include?(val) }
+      required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
+
+      # Consumer needs to be present only if topic is active
+      # We allow not to define consumer for non-active because they may be only used via admin
+      # api or other ways and not consumed with consumer
+      virtual do |data, errors|
+        next unless errors.empty?
+        next if data.fetch(:consumer)
+        next unless data.fetch(:active)
+
+        [[%w[consumer], :missing]]
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        value = data.fetch(:kafka)
+
+        begin
+          # This will trigger rdkafka validations that we catch and re-map the info and use dry
+          # compatible format
+          Rdkafka::Config.new(value).send(:native_config)
+
+          nil
+        rescue Rdkafka::Config::ConfigError => e
+          [[%w[kafka], e.message]]
+        end
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
+
+        value = data.fetch(:name)
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+
+        next if namespacing_chars_count <= 1
+
+        [[%w[name], :inconsistent_namespacing]]
+      end
+    end
+  end
+end

data/lib/karafka/contracts.rb

@@ -5,6 +5,6 @@ module Karafka
   module Contracts
     # Regexp for validating format of groups and topics
     # @note It is not nested inside of the contracts, as it is used by couple of them
-    TOPIC_REGEXP = /\A(\w|-|\.)+\z/
+    TOPIC_REGEXP = /^[A-Za-z0-9\-_.]+$/
   end
 end

data/lib/karafka/embedded.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Allows to start and stop Karafka as part of a different process
+  module Embedded
+    class << self
+      # Starts Karafka without supervision and without ownership of signals in a background thread
+      # so it won't interrupt other things running
+      def start
+        Thread.new do
+          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Server.start
+        end
+      end
+
+      # Stops Karafka upon any event
+      #
+      # @note This method is blocking because we want to wait until Karafka is stopped with final
+      #   process shutdown
+      def stop
+        # Stop needs to be blocking to wait for all the things to finalize
+        Karafka::Server.stop
+      end
+
+      # Quiets Karafka upon any event
+      #
+      # @note This method is not blocking and will not wait for Karafka to fully quiet.
+      # It will trigger the quiet procedure but won't wait.
+      #
+      # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
+      def quiet
+        Karafka::Server.quiet
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -41,11 +41,13 @@ module Karafka
     # Raised when the license token is not valid
     InvalidLicenseTokenError = Class.new(BaseError)
 
-    # Used to instrument this error into the error notifications
-    # We do not raise it so we won't crash deployed systems
-    ExpiredLicenseTokenError = Class.new(BaseError)
+    # This should never happen. Please open an issue if it does.
+    InvalidCoordinatorStateError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    StrategyNotFoundError = Class.new(BaseError)
 
     # This should never happen. Please open an issue if it does.
-    InvalidCoordinatorState = Class.new(BaseError)
+    InvalidRealOffsetUsage = Class.new(BaseError)
   end
 end

data/lib/karafka/helpers/colorize.rb

@@ -15,6 +15,12 @@ module Karafka
       def red(string)
         "\033[0;31m#{string}\033[0m"
       end
+
+      # @param string [String] string we want to have in yellow
+      # @return [String] yellow string
+      def yellow(string)
+        "\033[1;33m#{string}\033[0m"
+      end
     end
   end
 end

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -9,12 +9,10 @@ module Karafka
         # @param subscription_group_id [String] id of the current subscription group instance
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
-        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
-        def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
+        def initialize(subscription_group_id, consumer_group_id, client_name)
           @subscription_group_id = subscription_group_id
           @consumer_group_id = consumer_group_id
           @client_name = client_name
-          @monitor = monitor
         end
 
         # Runs the instrumentation monitor with error
@@ -26,8 +24,9 @@ module Karafka
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
-          @monitor.instrument(
+          ::Karafka.monitor.instrument(
             'error.occurred',
+            caller: self,
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
             type: 'librdkafka.error',

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -10,12 +10,10 @@ module Karafka
         # @param subscription_group_id [String] id of the current subscription group
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
-        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
-        def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
+        def initialize(subscription_group_id, consumer_group_id, client_name)
           @subscription_group_id = subscription_group_id
           @consumer_group_id = consumer_group_id
           @client_name = client_name
-          @monitor = monitor
           @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
@@ -28,12 +26,24 @@ module Karafka
           # all the time.
           return unless @client_name == statistics['name']
 
-          @monitor.instrument(
+          ::Karafka.monitor.instrument(
             'statistics.emitted',
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
             statistics: @statistics_decorator.call(statistics)
           )
+        # We need to catch and handle any potential errors coming from the instrumentation pipeline
+        # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
+        # will hang the whole process.
+        rescue StandardError => e
+          ::Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            type: 'statistics.emitted.error',
+            error: e
+          )
         end
       end
     end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -14,9 +14,11 @@ module Karafka
         fatal
       ].freeze
 
+      private_constant :USED_LOG_LEVELS
+
       # Logs each messages fetching attempt
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
         listener = event[:caller]
         debug "[#{listener.id}] Polling messages..."
@@ -24,7 +26,7 @@ module Karafka
 
       # Logs about messages that we've received from Kafka
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop_received(event)
         listener = event[:caller]
         time = event[:time]
@@ -40,31 +42,79 @@ module Karafka
 
       # Prints info about the fact that a given job has started
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_process(event)
         job = event[:job]
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+        partition = job.executor.partition
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}/#{partition} started"
       end
 
       # Prints info about the fact that a given job has finished
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
         time = event[:time]
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
+        partition = job.executor.partition
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{job.id}] #{job_type} job for #{consumer}
+          on #{topic}/#{partition} finished in #{time}ms
+        MSG
+      end
+
+      # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_client_pause(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        client = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{client.id}] Pausing on topic #{topic}/#{partition} on offset #{offset}
+        MSG
+      end
+
+      # Prints information about resuming of processing of a given topic partition
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_client_resume(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        client = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{client.id}] Resuming on topic #{topic}/#{partition}
+        MSG
+      end
+
+      # Prints info about retry of processing after an error
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_retry(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        consumer = event[:caller]
+        timeout = event[:timeout]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
+          on topic #{topic}/#{partition} from offset #{offset}
+        MSG
       end
 
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_process_notice_signal(event)
         info "Received #{event[:signal]} system signal"
 
@@ -85,37 +135,96 @@ module Karafka
         end
       end
 
-      # Logs info that we're initializing Karafka app.
-      #
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_initializing(_event)
-        info 'Initializing Karafka framework'
-      end
-
       # Logs info that we're running Karafka app.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_running(_event)
-        info 'Running Karafka server'
+        info "Running in #{RUBY_DESCRIPTION}"
+        info "Running Karafka #{Karafka::VERSION} server"
+
+        return if Karafka.pro?
+
+        info 'See LICENSE and the LGPL-3.0 for licensing details'
+      end
+
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quieting(_event)
+        info 'Switching to quiet mode. New messages will not be processed'
+      end
+
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(_event)
+        info 'Reached quiet mode. No messages will be processed anymore'
       end
 
       # Logs info that we're going to stop the Karafka server.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopping(_event)
         info 'Stopping Karafka server'
       end
 
       # Logs info that we stopped the Karafka server.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopped(_event)
         info 'Stopped Karafka server'
       end
 
+      # Logs info when we have dispatched a message the the DLQ
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_dead_letter_queue_dispatched(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
+        message = event[:message]
+        offset = message.offset
+        dlq_topic = consumer.topic.dead_letter_queue.topic
+        partition = message.partition
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Dispatched message #{offset}
+          from #{topic}/#{partition}
+          to DLQ topic: #{dlq_topic}
+        MSG
+      end
+
+      # Logs info about throttling event
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_filtering_throttled(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
+        # Here we get last message before throttle
+        message = event[:message]
+        partition = message.partition
+        offset = message.offset
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Throttled and will resume
+          from message #{offset}
+          on #{topic}/#{partition}
+        MSG
+      end
+
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_filtering_seek(event)
+        consumer = event[:caller]
+        topic = consumer.topic.name
+        # Message to which we seek
+        message = event[:message]
+        partition = message.partition
+        offset = message.offset
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Post-filtering seeking to message #{offset}
+          on #{topic}/#{partition}
+        MSG
+      end
+
       # There are many types of errors that can occur in many places, but we provide a single
       # handler for all of them to simplify error instrumentation.
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_error_occurred(event)
         type = event[:type]
         error = event[:error]
@@ -128,6 +237,18 @@ module Karafka
         when 'consumer.revoked.error'
           error "Consumer on revoked failed due to an error: #{error}"
           error details
+        when 'consumer.before_enqueue.error'
+          error "Consumer before enqueue failed due to an error: #{error}"
+          error details
+        when 'consumer.before_consume.error'
+          error "Consumer before consume failed due to an error: #{error}"
+          error details
+        when 'consumer.after_consume.error'
+          error "Consumer after consume failed due to an error: #{error}"
+          error details
+        when 'consumer.idle.error'
+          error "Consumer idle failed due to an error: #{error}"
+          error details
         when 'consumer.shutdown.error'
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
@@ -137,9 +258,6 @@ module Karafka
         when 'connection.listener.fetch_loop.error'
           error "Listener fetch loop error: #{error}"
           error details
-        when 'licenser.expired'
-          error error
-          error details
         when 'runner.call.error'
           fatal "Runner crashed due to an error: #{error}"
           fatal details
@@ -148,6 +266,17 @@ module Karafka
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details
+        # Those can occur when emitted statistics are consumed by the end user and the processing
+        # of statistics fails. The statistics are emitted from librdkafka main loop thread and
+        # any errors there crash the whole thread
+        when 'statistics.emitted.error'
+          error "statistics.emitted processing failed due to an error: #{error}"
+          error details
+        # Those will only occur when retries in the client fail and when they did not stop after
+        # back-offs
+        when 'connection.client.poll.error'
+          error "Data polling error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/instrumentation/notifications.rb

@@ -17,31 +17,50 @@ module Karafka
       #   complete list of all the events. Please use the #available_events on fully loaded
       #   Karafka system to determine all of the events you can use.
       EVENTS = %w[
+        active_job.consume
+        active_job.consumed
+
         app.initialized
         app.running
+        app.quieting
+        app.quiet
         app.stopping
         app.stopped
+        app.terminated
+
+        client.pause
+        client.resume
+
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.received
 
+        consumer.consume
         consumer.consumed
+        consumer.consuming.pause
+        consumer.consuming.retry
+        consumer.idle
+        consumer.revoke
         consumer.revoked
+        consumer.shutting_down
         consumer.shutdown
 
+        dead_letter_queue.dispatched
+
+        filtering.throttled
+        filtering.seek
+
         process.notice_signal
 
-        connection.listener.before_fetch_loop
-        connection.listener.fetch_loop
-        connection.listener.fetch_loop.received
+        statistics.emitted
 
         worker.process
         worker.processed
-
-        statistics.emitted
+        worker.completed
 
         error.occurred
       ].freeze
 
-      private_constant :EVENTS
-
       # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,22 +4,13 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
-      # Updates proc title to an initializing one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_initializing(_event)
-        setproctitle('initializing')
-      end
-
-      # Updates proc title to a running one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        setproctitle('running')
-      end
-
-      # Updates proc title to a stopping one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        setproctitle('stopping')
+      Status::STATES.each_key do |state|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          # Updates proc title to an appropriate state
+          def on_app_#{state}(_event)
+            setproctitle('#{state}')
+          end
+        RUBY
       end
 
       private