karafka 1.4.12 → 2.2.10

data/lib/karafka/server.rb

@@ -3,8 +3,6 @@
 module Karafka
   # Karafka consuming server class
   class Server
-    @consumer_threads = Concurrent::Array.new
-
     # How long should we sleep between checks on shutting down consumers
     SUPERVISION_SLEEP = 0.1
     # What system exit code should we use when we terminated forcefully
@@ -17,69 +15,147 @@ module Karafka
 
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
-      attr_accessor :consumer_threads
+      attr_accessor :listeners
+
+      # Set of workers
+      attr_accessor :workers
 
-      # Writer for list of consumer groups that we want to consume in our current process context
-      attr_writer :consumer_groups
+      # Jobs queue
+      attr_accessor :jobs_queue
 
       # Method which runs app
       def run
-        process.on_sigint { stop_supervised }
-        process.on_sigquit { stop_supervised }
-        process.on_sigterm { stop_supervised }
-        run_supervised
-      end
-
-      # @return [Array<String>] array with names of consumer groups that should be consumed in a
-      #   current server context
-      def consumer_groups
-        # If not specified, a server will listed on all the topics
-        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
-      end
-
-      private
+        self.listeners = []
+        self.workers = []
+
+        # We need to validate this prior to running because it may be executed also from the
+        # embedded
+        # We cannot validate this during the start because config needs to be populated and routes
+        # need to be defined.
+        config.internal.cli.contract.validate!(
+          config.internal.routing.activity_manager.to_h
+        )
+
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+        process.on_sigtstp { quiet }
+        process.supervise
 
-      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
-      def process
-        Karafka::App.config.internal.process
+        # Start is blocking until stop is called and when we stop, it will wait until
+        # all of the things are ready to stop
+        start
+
+        # We always need to wait for Karafka to stop here since we should wait for the stop running
+        # in a separate thread (or trap context) to indicate everything is closed
+        # Since `#start` is blocking, we will get here only after the runner is done. This will
+        # not add any performance degradation because of that.
+        sleep(0.1) until Karafka::App.terminated?
+      # Try its best to shutdown underlying components before re-raising
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+        # rubocop:enable Lint/RescueException
+        stop
+
+        raise e
       end
 
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)
-      def run_supervised
-        process.supervise
+      def start
         Karafka::App.run!
-        Karafka::App.config.internal.fetcher.call
+        Karafka::Runner.new.call
       end
 
       # Stops Karafka with a supervision (as long as there is a shutdown timeout)
-      # If consumers won't stop in a given time frame, it will force them to exit
-      def stop_supervised
+      # If consumers or workers won't stop in a given time frame, it will force them to exit
+      #
+      # @note This method is not async. It should not be executed from the workers as it will
+      #   lock them forever. If you need to run Karafka shutdown from within workers threads,
+      #   please start a separate thread to do so.
+      def stop
+        # Initialize the stopping process only if Karafka was running
+        return if Karafka::App.stopping?
+        return if Karafka::App.stopped?
+        return if Karafka::App.terminated?
+
         Karafka::App.stop!
 
-        # See https://github.com/dry-rb/dry-configurable/issues/93
-        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+        timeout = config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
-        (timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if consumer_threads.count(&:alive?).zero?
-            Thread.new { Karafka.monitor.instrument('app.stopped') }.join
-            return
-          end
+        # We divide it by 1000 because we use time in ms.
+        ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
+          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
 
           sleep SUPERVISION_SLEEP
         end
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
-        Thread.new { Karafka.monitor.instrument('app.stopping.error', error: e) }.join
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'app.stopping.error'
+        )
+
         # We're done waiting, lets kill them!
-        consumer_threads.each(&:terminate)
+        workers.each(&:terminate)
+        listeners.each(&:terminate)
+        # We always need to shutdown clients to make sure we do not force the GC to close consumer.
+        # This can cause memory leaks and crashes.
+        listeners.each(&:shutdown)
+
+        # We also do not forcefully terminate everything when running in the embedded mode,
+        # otherwise we would overwrite the shutdown process of the process that started Karafka
+        return unless process.supervised?
 
         # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit! FORCEFUL_EXIT_CODE
+        Kernel.exit!(FORCEFUL_EXIT_CODE)
+      ensure
+        # We need to check if it wasn't an early exit to make sure that only on stop invocation
+        # can change the status after everything is closed
+        if timeout
+          Karafka::App.stopped!
+
+          # We close producer as the last thing as it can be used in the notification pipeline
+          # to dispatch state changes, etc
+          Karafka::App.producer.close
+
+          Karafka::App.terminate!
+        end
+      end
+
+      # Quiets the Karafka server.
+      #
+      # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
+      # triggered until final shutdown.
+      def quiet
+        # We don't have to safe-guard it with check states as the state transitions work only
+        # in one direction
+        Karafka::App.quiet!
+
+        # We need one more thread to monitor the process and move to quieted once everything
+        # is quiet and no processing is happening anymore
+        Thread.new do
+          sleep(0.1) until listeners.coordinators.all?(&:finished?)
+          Karafka::App.quieted!
+        end
+      end
+
+      private
+
+      # @return [Karafka::Core::Configurable::Node] root config node
+      def config
+        Karafka::App.config
+      end
+
+      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
+      def process
+        config.internal.process
       end
     end
   end

data/lib/karafka/setup/attributes_map.rb

@@ -0,0 +1,348 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # To simplify the overall design, in Karafka we define all the rdkafka settings in one scope
+    # under `kafka`. rdkafka though does not like when producer options are passed to the
+    # consumer configuration and issues warnings. This target map is used as a filtering layer, so
+    # only appropriate settings go to both producer and consumer
+    #
+    # It is built based on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    module AttributesMap
+      # List of rdkafka consumer accepted attributes
+      CONSUMER = %i[
+        allow.auto.create.topics
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        auto.commit.enable
+        auto.commit.interval.ms
+        auto.offset.reset
+        background_event_cb
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        check.crcs
+        client.dns.lookup
+        client.id
+        client.rack
+        closesocket_cb
+        connect_cb
+        connections.max.idle.ms
+        consume.callback.max.messages
+        consume_cb
+        coordinator.query.interval.ms
+        debug
+        default_topic_conf
+        enable.auto.commit
+        enable.auto.offset.store
+        enable.partition.eof
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        fetch.error.backoff.ms
+        fetch.max.bytes
+        fetch.message.max.bytes
+        fetch.min.bytes
+        fetch.queue.backoff.ms
+        fetch.wait.max.ms
+        group.id
+        group.instance.id
+        group.protocol.type
+        heartbeat.interval.ms
+        interceptors
+        internal.termination.signal
+        isolation.level
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        max.partition.fetch.bytes
+        max.poll.interval.ms
+        message.copy.max.bytes
+        message.max.bytes
+        metadata.broker.list
+        metadata.max.age.ms
+        oauthbearer_token_refresh_cb
+        offset.store.method
+        offset.store.path
+        offset.store.sync.interval.ms
+        offset_commit_cb
+        opaque
+        open_cb
+        partition.assignment.strategy
+        plugin.library.paths
+        queued.max.messages.kbytes
+        queued.min.messages
+        rebalance_cb
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        resolve_cb
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        session.timeout.ms
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.providers
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+      ].freeze
+
+      # List of rdkafka producer accepted attributes
+      PRODUCER = %i[
+        acks
+        api.version.fallback.ms
+        api.version.request
+        api.version.request.timeout.ms
+        background_event_cb
+        batch.num.messages
+        batch.size
+        bootstrap.servers
+        broker.address.family
+        broker.address.ttl
+        broker.version.fallback
+        builtin.features
+        client.dns.lookup
+        client.id
+        client.rack
+        closesocket_cb
+        compression.codec
+        compression.level
+        compression.type
+        connect_cb
+        connections.max.idle.ms
+        debug
+        default_topic_conf
+        delivery.report.only.error
+        delivery.timeout.ms
+        dr_cb
+        dr_msg_cb
+        enable.gapless.guarantee
+        enable.idempotence
+        enable.random.seed
+        enable.sasl.oauthbearer.unsecure.jwt
+        enable.ssl.certificate.verification
+        enabled_events
+        error_cb
+        interceptors
+        internal.termination.signal
+        linger.ms
+        log.connection.close
+        log.queue
+        log.thread.name
+        log_cb
+        log_level
+        max.in.flight
+        max.in.flight.requests.per.connection
+        message.copy.max.bytes
+        message.max.bytes
+        message.send.max.retries
+        message.timeout.ms
+        metadata.broker.list
+        metadata.max.age.ms
+        msg_order_cmp
+        oauthbearer_token_refresh_cb
+        opaque
+        open_cb
+        partitioner
+        partitioner_cb
+        plugin.library.paths
+        produce.offset.report
+        queue.buffering.backpressure.threshold
+        queue.buffering.max.kbytes
+        queue.buffering.max.messages
+        queue.buffering.max.ms
+        queuing.strategy
+        receive.message.max.bytes
+        reconnect.backoff.jitter.ms
+        reconnect.backoff.max.ms
+        reconnect.backoff.ms
+        request.required.acks
+        request.timeout.ms
+        resolve_cb
+        retries
+        retry.backoff.max.ms
+        retry.backoff.ms
+        sasl.kerberos.keytab
+        sasl.kerberos.kinit.cmd
+        sasl.kerberos.min.time.before.relogin
+        sasl.kerberos.principal
+        sasl.kerberos.service.name
+        sasl.mechanism
+        sasl.mechanisms
+        sasl.oauthbearer.client.id
+        sasl.oauthbearer.client.secret
+        sasl.oauthbearer.config
+        sasl.oauthbearer.extensions
+        sasl.oauthbearer.method
+        sasl.oauthbearer.scope
+        sasl.oauthbearer.token.endpoint.url
+        sasl.password
+        sasl.username
+        security.protocol
+        socket.blocking.max.ms
+        socket.connection.setup.timeout.ms
+        socket.keepalive.enable
+        socket.max.fails
+        socket.nagle.disable
+        socket.receive.buffer.bytes
+        socket.send.buffer.bytes
+        socket.timeout.ms
+        socket_cb
+        ssl.ca.certificate.stores
+        ssl.ca.location
+        ssl.ca.pem
+        ssl.certificate.location
+        ssl.certificate.pem
+        ssl.certificate.verify_cb
+        ssl.cipher.suites
+        ssl.crl.location
+        ssl.curves.list
+        ssl.endpoint.identification.algorithm
+        ssl.engine.id
+        ssl.engine.location
+        ssl.key.location
+        ssl.key.password
+        ssl.key.pem
+        ssl.keystore.location
+        ssl.keystore.password
+        ssl.providers
+        ssl.sigalgs.list
+        ssl_ca
+        ssl_certificate
+        ssl_engine_callback_data
+        ssl_key
+        statistics.interval.ms
+        stats_cb
+        sticky.partitioning.linger.ms
+        throttle_cb
+        topic.blacklist
+        topic.metadata.propagation.max.ms
+        topic.metadata.refresh.fast.cnt
+        topic.metadata.refresh.fast.interval.ms
+        topic.metadata.refresh.interval.ms
+        topic.metadata.refresh.sparse
+        transaction.timeout.ms
+        transactional.id
+      ].freeze
+
+      # Location of the file with rdkafka settings list
+      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+
+      private_constant :SOURCE
+
+      class << self
+        # Filter the provided settings leaving only the once applicable to the consumer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the consumer
+        def consumer(kafka_settings)
+          kafka_settings.slice(*CONSUMER)
+        end
+
+        # Filter the provided settings leaving only the once applicable to the producer
+        # @param kafka_settings [Hash] all kafka settings
+        # @return [Hash] settings applicable to the producer
+        def producer(kafka_settings)
+          kafka_settings.slice(*PRODUCER)
+        end
+
+        # @private
+        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        #   that is sorted.
+        # @note This method should not be used directly. It is only used to generate appropriate
+        #   options list in case it would change
+        def generate
+          # Not used anywhere else, hence required here
+          require 'open-uri'
+
+          attributes = { consumer: Set.new, producer: Set.new }
+
+          ::URI.parse(SOURCE).open.readlines.each do |line|
+            next unless line.include?('|')
+
+            attribute, attribute_type = line.split('|').map(&:strip)
+
+            case attribute_type
+            when 'C'
+              attributes[:consumer] << attribute
+            when 'P'
+              attributes[:producer] << attribute
+            when '*'
+              attributes[:consumer] << attribute
+              attributes[:producer] << attribute
+            else
+              next
+            end
+          end
+
+          # This can be removed when 0.13 librdkafka is released
+          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
+
+          attributes.transform_values!(&:sort)
+          attributes.each_value { |vals| vals.map!(&:to_sym) }
+          attributes
+        end
+      end
+    end
+  end
+end