karafka 1.4.13 → 2.0.0

data/lib/active_job/karafka.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_job'
+  require_relative 'queue_adapters/karafka_adapter'
+
+  module ActiveJob
+    # Namespace for usage simplification outside of Rails where Railtie will not kick in.
+    # That way a require 'active_job/karafka' should be enough to use it
+    module Karafka
+    end
+  end
+
+  # We extend routing builder by adding a simple wrapper for easier jobs topics defining
+  # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
+  # the railtie initializer would be too late
+  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::Routing::Extensions
+  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::Routing::Extensions
+rescue LoadError
+  # We extend ActiveJob stuff in the railtie
+end

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# ActiveJob components to allow for jobs consumption with Karafka
+module ActiveJob
+  # ActiveJob queue adapters
+  module QueueAdapters
+    # Karafka adapter for enqueuing jobs
+    # This is here for ease of integration with ActiveJob.
+    class KarafkaAdapter
+      # Enqueues the job using the configured dispatcher
+      #
+      # @param job [Object] job that should be enqueued
+      def enqueue(job)
+        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
+      end
+
+      # Raises info, that Karafka backend does not support scheduling jobs
+      #
+      # @param _job [Object] job we cannot enqueue
+      # @param _timestamp [Time] time when job should run
+      def enqueue_at(_job, _timestamp)
+        raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
+      end
+    end
+  end
+end

data/lib/karafka/active_job/consumer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
+    # It marks the offset after each message, so we make sure, none of the jobs is executed twice
+    class Consumer < ::Karafka::BaseConsumer
+      # Executes the ActiveJob logic
+      # @note ActiveJob does not support batches, so we just run one message after another
+      def consume
+        messages.each do |message|
+          break if Karafka::App.stopping?
+
+          ::ActiveJob::Base.execute(
+            # We technically speaking could set this as deserializer and reference it from the
+            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+            # the ActiveJob setup here
+            ::ActiveSupport::JSON.decode(message.raw_payload)
+          )
+
+          mark_as_consumed(message)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+    class Dispatcher
+      # Defaults for dispatching
+      # The can be updated by using `#karafka_options` on the job
+      DEFAULTS = {
+        dispatch_method: :produce_async
+      }.freeze
+
+      private_constant :DEFAULTS
+
+      # @param job [ActiveJob::Base] job
+      def call(job)
+        ::Karafka.producer.public_send(
+          fetch_option(job, :dispatch_method, DEFAULTS),
+          topic: job.queue_name,
+          payload: ::ActiveSupport::JSON.encode(job.serialize)
+        )
+      end
+
+      private
+
+      # @param job [ActiveJob::Base] job
+      # @param key [Symbol] key we want to fetch
+      # @param defaults [Hash]
+      # @return [Object] options we are interested in
+      def fetch_option(job, key, defaults)
+        job
+          .class
+          .karafka_options
+          .fetch(key, defaults.fetch(key))
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_extensions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Allows for setting karafka specific options in ActiveJob jobs
+    module JobExtensions
+      class << self
+        # Defines all the needed accessors and sets defaults
+        # @param klass [ActiveJob::Base] active job base
+        def extended(klass)
+          klass.class_attribute :_karafka_options
+          klass._karafka_options = {}
+        end
+      end
+
+      # @param new_options [Hash] additional options that allow for jobs Karafka related options
+      #   customization
+      # @return [Hash] karafka options
+      def karafka_options(new_options = {})
+        return _karafka_options if new_options.empty?
+
+        # Make sure, that karafka options that someone wants to use are valid before assigning
+        # them
+        App.config.internal.active_job.job_options_contract.validate!(new_options)
+
+        new_options.each do |name, value|
+          _karafka_options[name] = value
+        end
+
+        _karafka_options
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_options_contract.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Contract for validating the options that can be altered with `#karafka_options` per job class
+    # @note We keep this in the `Karafka::ActiveJob` namespace instead of `Karafka::Contracts` as
+    #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
+    #   all in the same place
+    class JobOptionsContract < Contracts::Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('job_options')
+      end
+
+      optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+    end
+  end
+end

data/lib/karafka/active_job/routing/extensions.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # ActiveJob related Karafka stuff
+  module ActiveJob
+    # Karafka routing ActiveJob related components
+    module Routing
+      # Routing extensions for ActiveJob
+      module Extensions
+        # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
+        # the consumer class
+        # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+        # @param block [Proc] block that we can use for some extra configuration
+        def active_job_topic(name, &block)
+          topic(name) do
+            consumer App.config.internal.active_job.consumer_class
+
+            next unless block
+
+            instance_eval(&block)
+
+            # This is handled by our custom ActiveJob consumer
+            # Without this, default behaviour would cause messages to skip upon shutdown as the
+            # offset would be committed for the last message
+            manual_offset_management true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,31 +6,24 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
-      end
-
-      # @return [Karafka::Routing::Builder] consumers builder instance
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
       def consumer_groups
-        config.internal.routing_builder
+        config
+          .internal
+          .routing
+          .builder
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      def subscription_groups
+        consumer_groups
+          .active
+          .flat_map(&:subscription_groups)
       end
 
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
           App.config.internal.status.send(delegated)
@@ -42,7 +35,9 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)

data/lib/karafka/base_consumer.rb

@@ -4,48 +4,108 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend Forwardable
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    attr_accessor :topic
+    # @return [Karafka::Messages::Messages] current messages batch
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
+    # Can be used to run preparation code
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api. This can act as a hook when creating non-blocking
+    #   consumers and doing other advanced stuff
+    def on_before_consume; end
 
-      private delegated_method_name
-    end
+    # Executes the default consumer flow.
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      Karafka.monitor.instrument('consumer.consumed', caller: self) do
+        consume
+      end
 
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
+      coordinator.consumption(self).success!
+    rescue StandardError => e
+      coordinator.consumption(self).failure!
 
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.consume.error'
+      )
+    ensure
+      # We need to decrease number of jobs that this coordinator coordinates as it has finished
+      coordinator.decrement
     end
 
-    # Executes the default consumer flow.
-    def call
-      process
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api.
+    def on_after_consume
+      return if revoked?
+
+      if coordinator.success?
+        coordinator.pause_tracker.reset
+
+        # Mark as consumed only if manual offset management is not on
+        return if topic.manual_offset_management?
+
+        # We use the non-blocking one here. If someone needs the blocking one, can implement it
+        # with manual offset management
+        mark_as_consumed(messages.last)
+      else
+        pause(@seek_offset || messages.first.offset)
+      end
     end
 
-    private
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_revoked
+      coordinator.revoke
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+      Karafka.monitor.instrument('consumer.revoked', caller: self) do
+        revoked
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
     end
 
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+        shutdown
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
+    end
+
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
@@ -53,5 +113,95 @@ module Karafka
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # Marks message as consumed in an async way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
+    #
+    # @note We keep track of this offset in case we would mark as consumed and got error when
+    #   processing another message. In case like this we do not pause on the message we've already
+    #   processed but rather at the next one. This applies to both sync and async versions of this
+    #   method.
+    def mark_as_consumed(message)
+      unless client.mark_as_consumed(message)
+        coordinator.revoke
+
+        return false
+      end
+
+      @seek_offset = message.offset + 1
+
+      true
+    end
+
+    # Marks message as consumed in a sync way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
+    def mark_as_consumed!(message)
+      unless client.mark_as_consumed!(message)
+        coordinator.revoke
+
+        return false
+      end
+
+      @seek_offset = message.offset + 1
+
+      true
+    end
+
+    # Pauses processing on a given offset for the current topic partition
+    #
+    # After given partition is resumed, it will continue processing from the given offset
+    # @param offset [Integer] offset from which we want to restart the processing
+    # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
+    #   default exponential pausing strategy defined for retries
+    def pause(offset, timeout = nil)
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
+
+      client.pause(
+        messages.metadata.topic,
+        messages.metadata.partition,
+        offset
+      )
+    end
+
+    # Resumes processing of the current topic partition
+    def resume
+      # This is sufficient to expire a partition pause, as with it will be resumed by the listener
+      # thread before the next poll.
+      coordinator.pause_tracker.expire
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer] offset where we want to seek
+    def seek(offset)
+      client.seek(
+        Karafka::Messages::Seek.new(
+          messages.metadata.topic,
+          messages.metadata.partition,
+          offset
+        )
+      )
+    end
+
+    # @return [Boolean] true if partition was revoked from the current consumer
+    # @note We know that partition got revoked because when we try to mark message as consumed,
+    #   unless if is successful, it will return false
+    def revoked?
+      coordinator.revoked?
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param args [Array] All the arguments that Thor desc method accepts
-        def desc(*args)
-          @desc ||= args
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, *@desc
+          cli_class.desc name, @desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/info.rb

@@ -7,24 +7,58 @@ module Karafka
     class Info < Base
       desc 'Print configuration details and other options of your application'
 
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
+
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
-        info = [
-          "Karafka version: #{Karafka::VERSION}",
+        postfix = Karafka.pro? ? ' + Pro' : ''
+
+        [
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if Karafka.pro?
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -7,24 +7,23 @@ module Karafka
   class Cli < Thor
     # Install Karafka Cli action
     class Install < Base
+      include Helpers::Colorize
+
       desc 'Install all required things for Karafka application in current directory'
 
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
+        lib
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
       # @param args [Array] all the things that Thor CLI accepts
@@ -35,6 +34,7 @@ module Karafka
             Bundler.default_lockfile
           )
         ).dependencies
+
         @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
@@ -44,16 +44,25 @@ module Karafka
           FileUtils.mkdir_p Karafka.root.join(dir)
         end
 
+        puts
+        puts 'Installing Karafka framework...'
+        puts 'Ruby on Rails detected...' if rails?
+        puts
+
         INSTALL_FILES_MAP.each do |source, target|
-          target = Karafka.root.join(target)
+          pathed_target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
+
+          File.open(pathed_target, 'w') { |file| file.write(render) }
 
-          File.open(target, 'w') { |file| file.write(render) }
+          puts "#{green('Created')} #{target}"
         end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
       end
 
       # @return [Boolean] true if we have Rails loaded