karafka 2.0.0 → 2.1.0

data/lib/karafka/setup/attributes_map.rb

@@ -0,0 +1,344 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # To simplify the overall design, in Karafka we define all the rdkafka settings in one scope
+    # under `kafka`. rdkafka though does not like when producer options are passed to the
+    # consumer configuration and issues warnings. This target map is used as a filtering layer, so
+    # only appropriate settings go to both producer and consumer
+    #
+    # It is built based on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    module AttributesMap
+      # List of rdkafka consumer accepted attributes
+      CONSUMER = %i[
+        allow.auto.create.topics
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        auto.commit.enable
+        auto.commit.interval.ms
+        auto.offset.reset
+        background_event_cb
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        check.crcs
+        client.id
+        client.rack
+        closesocket_cb
+        connect_cb
+        connections.max.idle.ms
+        consume.callback.max.messages
+        consume_cb
+        coordinator.query.interval.ms
+        debug
+        default_topic_conf
+        enable.auto.commit
+        enable.auto.offset.store
+        enable.partition.eof
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        fetch.error.backoff.ms
+        fetch.max.bytes
+        fetch.message.max.bytes
+        fetch.min.bytes
+        fetch.wait.max.ms
+        group.id
+        group.instance.id
+        group.protocol.type
+        heartbeat.interval.ms
+        interceptors
+        internal.termination.signal
+        isolation.level
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        max.partition.fetch.bytes
+        max.poll.interval.ms
+        message.copy.max.bytes
+        message.max.bytes
+        metadata.broker.list
+        metadata.max.age.ms
+        oauthbearer_token_refresh_cb
+        offset.store.method
+        offset.store.path
+        offset.store.sync.interval.ms
+        offset_commit_cb
+        opaque
+        open_cb
+        partition.assignment.strategy
+        plugin.library.paths
+        queued.max.messages.kbytes
+        queued.min.messages
+        rebalance_cb
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        resolve_cb
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        session.timeout.ms
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.providers
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+      ].freeze
+
+      # List of rdkafka producer accepted attributes
+      PRODUCER = %i[
+        acks
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        background_event_cb
+        batch.num.messages
+        batch.size
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        client.id
+        client.rack
+        closesocket_cb
+        compression.codec
+        compression.level
+        compression.type
+        connect_cb
+        connections.max.idle.ms
+        debug
+        default_topic_conf
+        delivery.report.only.error
+        delivery.timeout.ms
+        dr_cb
+        dr_msg_cb
+        enable.gapless.guarantee
+        enable.idempotence
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        interceptors
+        internal.termination.signal
+        linger.ms
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        message.copy.max.bytes
+        message.max.bytes
+        message.send.max.retries
+        message.timeout.ms
+        metadata.broker.list
+        metadata.max.age.ms
+        msg_order_cmp
+        oauthbearer_token_refresh_cb
+        opaque
+        open_cb
+        partitioner
+        partitioner_cb
+        plugin.library.paths
+        produce.offset.report
+        queue.buffering.backpressure.threshold
+        queue.buffering.max.kbytes
+        queue.buffering.max.messages
+        queue.buffering.max.ms
+        queuing.strategy
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        request.required.acks
+        request.timeout.ms
+        resolve_cb
+        retries
+        retry.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.providers
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        sticky.partitioning.linger.ms
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+        transaction.timeout.ms
+        transactional.id
+      ].freeze
+
+      # Location of the file with rdkafka settings list
+      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+
+      private_constant :SOURCE
+
+      class << self
+        # Filter the provided settings leaving only the once applicable to the consumer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the consumer
+        def consumer(kafka_settings)
+          kafka_settings.slice(*CONSUMER)
+        end
+
+        # Filter the provided settings leaving only the once applicable to the producer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the producer
+        def producer(kafka_settings)
+          kafka_settings.slice(*PRODUCER)
+        end
+
+        # @private
+        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        #   that is sorted.
+        # @note This method should not be used directly. It is only used to generate appropriate
+        #   options list in case it would change
+        def generate
+          # Not used anywhere else, hence required here
+          require 'open-uri'
+
+          attributes = { consumer: Set.new, producer: Set.new }
+
+          ::URI.parse(SOURCE).open.readlines.each do |line|
+            next unless line.include?('|')
+
+            attribute, attribute_type = line.split('|').map(&:strip)
+
+            case attribute_type
+            when 'C'
+              attributes[:consumer] << attribute
+            when 'P'
+              attributes[:producer] << attribute
+            when '*'
+              attributes[:consumer] << attribute
+              attributes[:producer] << attribute
+            else
+              next
+            end
+          end
+
+          # This can be removed when 0.13 librdkafka is released
+          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
+
+          attributes.transform_values!(&:sort)
+          attributes.each_value { |vals| vals.map!(&:to_sym) }
+          attributes
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -16,11 +16,19 @@ module Karafka
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id': 'karafka'
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000,
+        'client.software.name': 'karafka',
+        'client.software.version': [
+          "v#{Karafka::VERSION}",
+          "rdkafka-ruby-v#{Rdkafka::VERSION}",
+          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
+        ].join('-')
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev
-      DEV_DEFAULTS = {
+      KAFKA_DEV_DEFAULTS = {
         # Will create non-existing topics automatically.
         # Note that the broker needs to be configured with `auto.create.topics.enable=true`
         # While it is not recommended in prod, it simplifies work in dev
@@ -32,7 +40,7 @@ module Karafka
         'topic.metadata.refresh.interval.ms': 5_000
       }.freeze
 
-      private_constant :KAFKA_DEFAULTS, :DEV_DEFAULTS
+      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
 
       # Available settings
 
@@ -68,8 +76,6 @@ module Karafka
       # option [String] should we start with the earliest possible offset or latest
       # This will set the `auto.offset.reset` value unless present in the kafka scope
       setting :initial_offset, default: 'earliest'
-      # option [Boolean] should we leave offset management to the user
-      setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
       setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
@@ -80,15 +86,20 @@ module Karafka
       setting :shutdown_timeout, default: 60_000
       # option [Integer] number of threads in which we want to do parallel processing
       setting :concurrency, default: 5
-      # option [Integer] how long should we wait upon processing error
+      # option [Integer] how long should we wait upon processing error (milliseconds)
       setting :pause_timeout, default: 1_000
-      # option [Integer] what is the max timeout in case of an exponential backoff
+      # option [Integer] what is the max timeout in case of an exponential backoff (milliseconds)
       setting :pause_max_timeout, default: 30_000
       # option [Boolean] should we use exponential backoff
       setting :pause_with_exponential_backoff, default: true
       # option [::WaterDrop::Producer, nil]
       # Unless configured, will be created once Karafka is configured based on user Karafka setup
       setting :producer, default: nil
+      # option [Boolean] when set to true, Karafka will ensure that the routing topic naming
+      # convention is strict
+      # Disabling this may be needed in scenarios where we do not have control over topics names
+      # and/or we work with existing systems where we cannot change topics names.
+      setting :strict_topics_namespacing, default: true
 
       # rdkafka default options
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
@@ -109,6 +120,10 @@ module Karafka
           # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
           #   group builder
           setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+          # Internally assigned list of limits on routings active for the current process
+          # This can be altered by the CLI command
+          setting :activity_manager, default: Routing::ActivityManager.new
         end
 
         setting :processing do
@@ -120,6 +135,14 @@ module Karafka
           setting :coordinator_class, default: Processing::Coordinator
           # option partitioner_class [Class] partitioner we use against a batch of data
           setting :partitioner_class, default: Processing::Partitioner
+          # option strategy_selector [Object] processing strategy selector to be used
+          setting :strategy_selector, default: Processing::StrategySelector.new
+        end
+
+        # Things related to operating on messages
+        setting :messages do
+          # Parser is used to convert raw payload prior to deserialization
+          setting :parser, default: Messages::Parser.new
         end
 
         # Karafka components for ActiveJob
@@ -142,17 +165,26 @@ module Karafka
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance
         def setup(&block)
+          # Will prepare and verify license if present
+          Licenser.prepare_and_verify(config.license)
+          # Will configure all the pro components
+          # This needs to happen before end user configuration as the end user may overwrite some
+          # of the pro defaults with custom components
+          Pro::Loader.pre_setup(config) if Karafka.pro?
+
           configure(&block)
           merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
-          licenser = Licenser.new
+          configure_components
 
-          # Tries to load our license gem and if present will try to load the correct license
-          licenser.prepare_and_verify(config.license)
+          # Refreshes the references that are cached that might have been changed by the config
+          ::Karafka.refresh!
 
-          configure_components
+          # Runs things that need to be executed after config is defined and all the components
+          # are also configured
+          Pro::Loader.post_setup(config) if Karafka.pro?
 
           Karafka::App.initialized!
         end
@@ -170,9 +202,12 @@ module Karafka
             config.kafka[key] = value
           end
 
+          # Use Karafka client_id as kafka client id if not set
+          config.kafka[:'client.id'] ||= config.client_id
+
           return if Karafka::App.env.production?
 
-          DEV_DEFAULTS.each do |key, value|
+          KAFKA_DEV_DEFAULTS.each do |key, value|
             next if config.kafka.key?(key)
 
             config.kafka[key] = value
@@ -185,15 +220,9 @@ module Karafka
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
-            producer_config.kafka = config.kafka.dup
+            producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
-
-          return unless Karafka.pro?
-
-          # Runs the pro loader that includes all the pro components
-          require 'karafka/pro/loader'
-          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/status.rb

@@ -8,30 +8,58 @@ module Karafka
       initializing: :initialize!,
       initialized: :initialized!,
       running: :run!,
+      # will no longer pickup any work, but current work will be finished
+      quieting: :quiet!,
+      # no work is happening but we keep process with the assignments running
+      quiet: :quieted!,
+      # shutdown started
       stopping: :stop!,
-      stopped: :stopped!
+      # all things are done and most of the things except critical are closed
+      stopped: :stopped!,
+      # immediately after this process exists
+      terminated: :terminate!
     }.freeze
 
-    private_constant :STATES
+    # Mutex to ensure that state transitions are thread-safe
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
 
     # By default we are in the initializing state
     def initialize
       initialize!
     end
 
+    # @return [String] stringified current app status
+    def to_s
+      @status.to_s
+    end
+
+    # Resets the status state
+    # This is used mostly in the integration suite
+    def reset!
+      @status = :initializing
+    end
+
     STATES.each do |state, transition|
       define_method :"#{state}?" do
         @status == state
       end
 
       define_method transition do
-        @status = state
+        MUTEX.synchronize do
+          # Do not allow reverse state transitions (we always go one way) or transition to the same
+          # state as currently
+          return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
+
+          @status = state
 
-        # Skip on creation (initializing)
-        # We skip as during this state we do not have yet a monitor
-        return if initializing?
+          # Skip on creation (initializing)
+          # We skip as during this state we do not have yet a monitor
+          return if initializing?
 
-        Karafka.monitor.instrument("app.#{state}")
+          Karafka.monitor.instrument("app.#{state}")
+        end
       end
     end
   end

data/lib/karafka/templates/karafka.rb.erb

@@ -1,14 +1,11 @@
 # frozen_string_literal: true
+<% unless rails? -%>
 
-<% if rails? -%>
-require ::File.expand_path('../config/environment', __FILE__)
-<% else -%>
 # This file is auto-generated during the install process.
 # If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
 # command again or refer to the install templates available in the source codes
 
-ENV['RACK_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
+ENV['KARAFKA_ENV'] ||= 'development'
 Bundler.require(:default, ENV['KARAFKA_ENV'])
 
 # Zeitwerk custom loader for loading the app components before the whole
@@ -43,13 +40,24 @@ class KarafkaApp < Karafka::App
   Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
+  )
+
   routes.draw do
 <% if rails? -%>
     # Uncomment this if you use Karafka with ActiveJob
-    # You ned to define the topic per each queue name you use
+    # You need to define the topic per each queue name you use
     # active_job_topic :default
 <% end -%>
     topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
       consumer ExampleConsumer
     end
   end

data/lib/karafka/time_trackers/base.rb

@@ -8,12 +8,7 @@ module Karafka
   module TimeTrackers
     # Base class for all the time-trackers.
     class Base
-      private
-
-      # @return [Float] current time in milliseconds
-      def now
-        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
-      end
+      include ::Karafka::Core::Helpers::Time
     end
   end
 end