karafka 1.4.12 → 2.2.10

data/lib/karafka/contracts/server_cli_options.rb

@@ -2,29 +2,90 @@
 
 module Karafka
   module Contracts
-    # Contract for validating correctness of the server cli command options
-    # We validate some basics + the list of consumer_groups on which we want to use, to make
-    # sure that all of them are defined, plus that a pidfile does not exist
-    class ServerCliOptions < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      params do
-        optional(:pid).filled(:str?)
-        optional(:daemon).filled(:bool?)
-        optional(:consumer_groups).value(:array, :filled?)
+    # Contract for validating correctness of the server cli command options.
+    class ServerCliOptions < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      rule(:pid) do
-        key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
-      end
+      %i[
+        include
+        exclude
+      ].each do |action|
+        optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+        optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+        optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_consumer_groups")
+
+          # If there were no consumer_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+          next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+          # Found unknown consumer groups
+          [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_subscription_groups")
+
+          # If there were no subscription_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          subscription_groups = Karafka::App
+                                .consumer_groups
+                                .map(&:subscription_groups)
+                                .flatten
+                                .map(&:name)
+
+          next if (value - subscription_groups).empty?
 
-      rule(:consumer_groups) do
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        if !value.nil? &&
-           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
-          key(:consumer_groups).failure(:consumer_groups_inclusion)
+          # Found unknown subscription groups
+          [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
         end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_topics")
+
+          # If there were no topics declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          topics = Karafka::App
+                   .consumer_groups
+                   .map(&:subscription_groups)
+                   .flatten
+                   .map(&:topics)
+                   .map { |gtopics| gtopics.map(&:name) }
+                   .flatten
+
+          next if (value - topics).empty?
+
+          # Found unknown topics
+          [[[:"#{action}_topics"], :topics_inclusion]]
+        end
+      end
+
+      # Makes sure we have anything to subscribe to when we start the server
+      virtual do |_, errors|
+        next unless errors.empty?
+
+        next unless Karafka::App.subscription_groups.empty?
+
+        [[%i[include_topics], :topics_missing]]
       end
     end
   end

data/lib/karafka/contracts/topic.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Consumer group topic validation rules.
+    class Topic < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('topic')
+      end
+
+      required(:deserializer) { |val| !val.nil? }
+      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:initial_offset) { |val| %w[earliest latest].include?(val) }
+      required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
+      required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:active) { |val| [true, false].include?(val) }
+      required(:subscription_group_name) { |val| val.is_a?(String) && !val.empty? }
+
+      # Consumer needs to be present only if topic is active
+      # We allow not to define consumer for non-active because they may be only used via admin
+      # api or other ways and not consumed with consumer
+      virtual do |data, errors|
+        next unless errors.empty?
+        next if data.fetch(:consumer)
+        next unless data.fetch(:active)
+
+        [[%w[consumer], :missing]]
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        value = data.fetch(:kafka)
+
+        begin
+          # This will trigger rdkafka validations that we catch and re-map the info and use dry
+          # compatible format
+          Rdkafka::Config.new(value).send(:native_config)
+
+          nil
+        rescue Rdkafka::Config::ConfigError => e
+          [[%w[kafka], e.message]]
+        end
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
+
+        value = data.fetch(:name)
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+
+        next if namespacing_chars_count <= 1
+
+        [[%w[name], :inconsistent_namespacing]]
+      end
+    end
+  end
+end

data/lib/karafka/contracts.rb

@@ -5,6 +5,6 @@ module Karafka
   module Contracts
     # Regexp for validating format of groups and topics
     # @note It is not nested inside of the contracts, as it is used by couple of them
-    TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
+    TOPIC_REGEXP = /^[A-Za-z0-9\-_.]+$/
   end
 end

data/lib/karafka/embedded.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Allows to start and stop Karafka as part of a different process
+  module Embedded
+    class << self
+      # Starts Karafka without supervision and without ownership of signals in a background thread
+      # so it won't interrupt other things running
+      def start
+        Thread.new do
+          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Server.start
+        end
+      end
+
+      # Stops Karafka upon any event
+      #
+      # @note This method is blocking because we want to wait until Karafka is stopped with final
+      #   process shutdown
+      def stop
+        # Stop needs to be blocking to wait for all the things to finalize
+        Karafka::Server.stop
+      end
+
+      # Quiets Karafka upon any event
+      #
+      # @note This method is not blocking and will not wait for Karafka to fully quiet.
+      # It will trigger the quiet procedure but won't wait.
+      #
+      # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
+      def quiet
+        Karafka::Server.quiet
+      end
+    end
+  end
+end

data/lib/karafka/env.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Env management class to get and set environment for Karafka
+  class Env < String
+    # Keys where we look for environment details for Karafka
+    LOOKUP_ENV_KEYS = %w[
+      KARAFKA_ENV
+      RACK_ENV
+      RAILS_ENV
+    ].freeze
+
+    # Default fallback env
+    DEFAULT_ENV = 'development'
+
+    private_constant :LOOKUP_ENV_KEYS, :DEFAULT_ENV
+
+    # @return [Karafka::Env] env object
+    # @note Will load appropriate environment automatically
+    def initialize
+      super('')
+
+      LOOKUP_ENV_KEYS
+        .map { |key| ENV[key] }
+        .compact
+        .first
+        .then { |env| env || DEFAULT_ENV }
+        .then { |env| replace(env) }
+    end
+
+    # @param method_name [String] method name
+    # @param include_private [Boolean] should we include private methods as well
+    # @return [Boolean] true if we respond to a given missing method, otherwise false
+    def respond_to_missing?(method_name, include_private = false)
+      (method_name[-1] == '?') || super
+    end
+
+    # Reacts to missing methods, from which some might be the env checks.
+    # If the method ends with '?' we assume, that it is an env check
+    # @param method_name [String] method name for missing or env name with question mark
+    # @param arguments [Array] any arguments that we pass to the method
+    def method_missing(method_name, *arguments)
+      method_name[-1] == '?' ? self == method_name[0..-2] : super
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -6,12 +6,6 @@ module Karafka
     # Base class for all the Karafka internal errors
     BaseError = Class.new(StandardError)
 
-    # Should be raised when we have that that we cannot serialize
-    SerializationError = Class.new(BaseError)
-
-    # Should be raised when we tried to deserialize incoming data but we failed
-    DeserializationError = Class.new(BaseError)
-
     # Raised when router receives topic name which does not correspond with any routes
     # This can only happen in a case when:
     #   - you've received a message and we cannot match it with a consumer
@@ -24,28 +18,50 @@ module Karafka
     # @see https://github.com/karafka/karafka/issues/135
     NonMatchingRouteError = Class.new(BaseError)
 
-    # Raised when we don't use or use responder not in the way it expected to based on the
-    # topics usage definitions
-    InvalidResponderUsageError = Class.new(BaseError)
-
-    # Raised when options that we provide to the responder to respond aren't what the contract
-    # requires
-    InvalidResponderMessageOptionsError = Class.new(BaseError)
-
     # Raised when configuration doesn't match with validation contract
     InvalidConfigurationError = Class.new(BaseError)
 
     # Raised when we try to use Karafka CLI commands (except install) without a boot file
     MissingBootFileError = Class.new(BaseError)
 
-    # Raised when we want to read a persisted thread messages consumer but it is unavailable
-    # This should never happen and if it does, please contact us
-    MissingClientError = Class.new(BaseError)
-
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEventError = Class.new(BaseError)
-
     # Raised when we've waited enough for shutting down a non-responsive process
     ForcefulShutdownError = Class.new(BaseError)
+
+    # Raised when the jobs queue receives a job that should not be received as it would cause
+    # the processing to go out of sync. We should never process in parallel data from the same
+    # topic partition (unless virtual partitions apply)
+    JobsQueueSynchronizationError = Class.new(BaseError)
+
+    # Raised when given topic is not found while expected
+    TopicNotFoundError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    UnsupportedCaseError = Class.new(BaseError)
+
+    # Raised when the license token is not valid
+    InvalidLicenseTokenError = Class.new(BaseError)
+
+    # Raised on attempt to deserializer a cleared message
+    MessageClearedError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidCoordinatorStateError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    StrategyNotFoundError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidRealOffsetUsageError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidTimeBasedOffsetError = Class.new(BaseError)
+
+    # For internal usage only
+    # Raised when we run operations that require certain result but despite successfully finishing
+    # it is not yet available due to some synchronization mechanisms and caches
+    ResultNotVisibleError = Class.new(BaseError)
+
+    # Raised when there is an attempt to run an unrecognized CLI command
+    UnrecognizedCommandError = Class.new(BaseError)
   end
 end

data/lib/karafka/helpers/async.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Allows a given class to run async in a separate thread. Provides also few methods we may
+    # want to use to control the underlying thread
+    #
+    # @note Thread running code needs to manage it's own exceptions. If they leak out, they will
+    #   abort thread on exception.
+    module Async
+      class << self
+        # Adds forwardable to redirect thread-based control methods to the underlying thread that
+        # runs the async operations
+        #
+        # @param base [Class] class we're including this module in
+        def included(base)
+          base.extend ::Forwardable
+
+          base.def_delegators :@thread, :join, :terminate, :alive?
+        end
+      end
+
+      # Runs the `#call` method in a new thread
+      def async_call
+        @thread = Thread.new do
+          Thread.current.abort_on_exception = true
+
+          call
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/colorize.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Simple wrapper for adding colors to strings
+    module Colorize
+      # @param string [String] string we want to have in green
+      # @return [String] green string
+      def green(string)
+        "\033[0;32m#{string}\033[0m"
+      end
+
+      # @param string [String] string we want to have in red
+      # @return [String] red string
+      def red(string)
+        "\033[0;31m#{string}\033[0m"
+      end
+
+      # @param string [String] string we want to have in yellow
+      # @return [String] yellow string
+      def yellow(string)
+        "\033[1;33m#{string}\033[0m"
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Module containing classes and methods that provide some additional functionalities
+  # Module containing classes and methods that provide some additional helper functionalities.
   module Helpers
     # @note Taken from http://stackoverflow.com/questions/6407141
-    # Multidelegator is used to delegate calls to multiple targets
+    # Multidelegator is used to delegate calls to multiple targets.
     class MultiDelegator
       # @param targets to which we want to delegate methods
       def initialize(*targets)

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Callbacks used to transport things from rdkafka
+    module Callbacks
+      # Callback that kicks in when consumer error occurs and is published in a background thread
+      class Error
+        # @param subscription_group_id [String] id of the current subscription group instance
+        # @param consumer_group_id [String] id of the current consumer group
+        # @param client_name [String] rdkafka client name
+        def initialize(subscription_group_id, consumer_group_id, client_name)
+          @subscription_group_id = subscription_group_id
+          @consumer_group_id = consumer_group_id
+          @client_name = client_name
+        end
+
+        # Runs the instrumentation monitor with error
+        # @param client_name [String] rdkafka client name
+        # @param error [Rdkafka::Error] error that occurred
+        # @note It will only instrument on errors of the client of our consumer
+        def call(client_name, error)
+          # Emit only errors related to our client
+          # Same as with statistics (mor explanation there)
+          return unless @client_name == client_name
+
+          ::Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            type: 'librdkafka.error',
+            error: error
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Callback that connects to the librdkafka rebalance callback and converts those events into
+      # our internal events
+      class Rebalance
+        # @param subscription_group_id [String] id of the current subscription group instance
+        # @param consumer_group_id [String] id of the current consumer group
+        def initialize(subscription_group_id, consumer_group_id)
+          @subscription_group_id = subscription_group_id
+          @consumer_group_id = consumer_group_id
+        end
+
+        # Publishes an event that partitions are going to be revoked.
+        # At this stage we can still commit offsets, etc.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_revoke(tpl)
+          instrument('partitions_revoke', tpl)
+        end
+
+        # Publishes an event that partitions are going to be assigned
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_assign(tpl)
+          instrument('partitions_assign', tpl)
+        end
+
+        # Publishes an event that partitions were revoked. This is after we've lost them, so no
+        # option to commit offsets.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_revoked(tpl)
+          instrument('partitions_revoked', tpl)
+        end
+
+        # Publishes an event that partitions were assigned.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_assigned(tpl)
+          instrument('partitions_assigned', tpl)
+        end
+
+        private
+
+        # Publishes info that a rebalance event of a given type has happened
+        #
+        # @param name [String] name of the event
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def instrument(name, tpl)
+          ::Karafka.monitor.instrument(
+            "rebalance.#{name}",
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            tpl: tpl
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Statistics callback handler
+      # @see `WaterDrop::Instrumentation::Callbacks::Statistics` for details on why we decorate
+      #   those statistics
+      class Statistics
+        # @param subscription_group_id [String] id of the current subscription group
+        # @param consumer_group_id [String] id of the current consumer group
+        # @param client_name [String] rdkafka client name
+        def initialize(subscription_group_id, consumer_group_id, client_name)
+          @subscription_group_id = subscription_group_id
+          @consumer_group_id = consumer_group_id
+          @client_name = client_name
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
+        end
+
+        # Emits decorated statistics to the monitor
+        # @param statistics [Hash] rdkafka statistics
+        def call(statistics)
+          # Emit only statistics related to our client
+          # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
+          # emit only stats that are related to current producer. Otherwise we would emit all of
+          # all the time.
+          return unless @client_name == statistics['name']
+
+          ::Karafka.monitor.instrument(
+            'statistics.emitted',
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            statistics: @statistics_decorator.call(statistics)
+          )
+        # We need to catch and handle any potential errors coming from the instrumentation pipeline
+        # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
+        # will hang the whole process.
+        rescue StandardError => e
+          ::Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            type: 'statistics.emitted.error',
+            error: e
+          )
+        end
+      end
+    end
+  end
+end