karafka 1.4.10 → 2.0.0.alpha2

data/karafka.gemspec

@@ -5,28 +5,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
 
-# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
-  spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
-  spec.homepage    = 'https://github.com/karafka/karafka'
+  spec.authors     = ['Maciej Mensfeld']
+  spec.email       = %w[maciej@mensfeld.pl]
+  spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.license     = 'MIT'
+  spec.license     = 'LGPL-3.0'
 
   spec.add_dependency 'dry-configurable', '~> 0.13'
-  spec.add_dependency 'dry-inflector', '~> 0.2'
   spec.add_dependency 'dry-monitor', '~> 0.5'
   spec.add_dependency 'dry-validation', '~> 1.7'
-  spec.add_dependency 'envlogic', '~> 1.1'
-  spec.add_dependency 'irb', '~> 1.3'
-  spec.add_dependency 'ruby-kafka', '>= 1.3.0'
-  spec.add_dependency 'thor', '>= 1.1'
-  spec.add_dependency 'waterdrop', '~> 1.4'
-  spec.add_dependency 'zeitwerk', '~> 2.4'
+  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'thor', '>= 0.20'
+  spec.add_dependency 'waterdrop', '>= 2.2.0', '< 3.0.0'
+  spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'
 
@@ -36,7 +32,11 @@ Gem::Specification.new do |spec|
 
   spec.cert_chain    = %w[certs/mensfeld.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/karafka',
+    'rubygems_mfa_required' => 'true'
+  }
 end
-# rubocop:enable Metrics/BlockLength

data/lib/active_job/karafka.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'active_job'
+require 'active_job/queue_adapters'
+require 'active_job/queue_adapters/karafka_adapter'
+
+module ActiveJob
+  # Namespace for usage simplification outside of Rails where Railtie will not kick in.
+  # That way a require 'active_job/karafka' should be enough to use it
+  module Karafka
+  end
+end
+
+# We extend routing builder by adding a simple wrapper for easier jobs topics defining
+# This needs to be extended here as it is going to be used in karafka routes, hence doing that in
+# the railtie initializer would be too late
+::Karafka::Routing::Builder.include ::Karafka::ActiveJob::RoutingExtensions
+::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::RoutingExtensions
+
+# We extend ActiveJob stuff in the railtie

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# ActiveJob components to allow for jobs consumption with Karafka
+module ActiveJob
+  # ActiveJob queue adapters
+  module QueueAdapters
+    # Karafka adapter for enqueuing jobs
+    # This is here for ease of integration with ActiveJob.
+    class KarafkaAdapter
+      # Enqueues the job using the configured dispatcher
+      #
+      # @param job [Object] job that should be enqueued
+      def enqueue(job)
+        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
+      end
+
+      # Raises info, that Karafka backend does not support scheduling jobs
+      #
+      # @param _job [Object] job we cannot enqueue
+      # @param _timestamp [Time] time when job should run
+      def enqueue_at(_job, _timestamp)
+        raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
+      end
+    end
+  end
+end

data/lib/karafka/active_job/consumer.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
+    # It marks the offset after each message, so we make sure, none of the jobs is executed twice
+    class Consumer < ::Karafka::BaseConsumer
+      # Executes the ActiveJob logic
+      # @note ActiveJob does not support batches, so we just run one message after another
+      def consume
+        messages.each do |message|
+          ::ActiveJob::Base.execute(
+            # We technically speaking could set this as deserializer and reference it from the
+            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+            # the ActiveJob setup here
+            ::ActiveSupport::JSON.decode(message.raw_payload)
+          )
+
+          mark_as_consumed(message)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+    class Dispatcher
+      # Defaults for dispatching
+      # The can be updated by using `#karafka_options` on the job
+      DEFAULTS = {
+        dispatch_method: :produce_async
+      }.freeze
+
+      private_constant :DEFAULTS
+
+      # @param job [ActiveJob::Base] job
+      def call(job)
+        ::Karafka.producer.public_send(
+          fetch_option(job, :dispatch_method, DEFAULTS),
+          topic: job.queue_name,
+          payload: ::ActiveSupport::JSON.encode(job.serialize)
+        )
+      end
+
+      private
+
+      # @param job [ActiveJob::Base] job
+      # @param key [Symbol] key we want to fetch
+      # @param defaults [Hash]
+      # @return [Object] options we are interested in
+      def fetch_option(job, key, defaults)
+        job
+          .class
+          .karafka_options
+          .fetch(key, defaults.fetch(key))
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_extensions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Allows for setting karafka specific options in ActiveJob jobs
+    module JobExtensions
+      class << self
+        # Defines all the needed accessors and sets defaults
+        # @param klass [ActiveJob::Base] active job base
+        def extended(klass)
+          klass.class_attribute :_karafka_options
+          klass._karafka_options = {}
+        end
+      end
+
+      # @param new_options [Hash] additional options that allow for jobs Karafka related options
+      #   customization
+      # @return [Hash] karafka options
+      def karafka_options(new_options = {})
+        return _karafka_options if new_options.empty?
+
+        # Make sure, that karafka options that someone wants to use are valid before assigning
+        # them
+        App.config.internal.active_job.job_options_contract.validate!(new_options)
+
+        new_options.each do |name, value|
+          _karafka_options[name] = value
+        end
+
+        _karafka_options
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_options_contract.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Contract for validating the options that can be altered with `#karafka_options` per job class
+    # @note We keep this in the `Karafka::ActiveJob` namespace instead of `Karafka::Contracts` as
+    #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
+    #   all in the same place
+    class JobOptionsContract < Contracts::Base
+      params do
+        optional(:dispatch_method).value(included_in?: %i[produce_async produce_sync])
+      end
+    end
+  end
+end

data/lib/karafka/active_job/routing_extensions.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  # ActiveJob related Karafka stuff
+  module ActiveJob
+    # Routing extensions for ActiveJob
+    module RoutingExtensions
+      # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
+      # the consumer class
+      # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+      def active_job_topic(name)
+        topic(name) do
+          consumer App.config.internal.active_job.consumer
+        end
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,31 +6,23 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
-      end
-
-      # @return [Karafka::Routing::Builder] consumers builder instance
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
       def consumer_groups
-        config.internal.routing_builder
+        config
+          .internal
+          .routing_builder
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      def subscription_groups
+        consumer_groups
+          .active
+          .flat_map(&:subscription_groups)
       end
 
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
           App.config.internal.status.send(delegated)
@@ -42,7 +34,9 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)

data/lib/karafka/base_consumer.rb

@@ -4,48 +4,80 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend Forwardable
-
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
-
-      private delegated_method_name
-    end
-
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
-
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
-    end
+    attr_accessor :topic
+    # @return [Karafka::Messages::Messages] current messages batch
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::TimeTrackers::Pause] current topic partition pause
+    attr_accessor :pause
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
     # Executes the default consumer flow.
-    def call
-      process
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      Karafka.monitor.instrument('consumer.consume', caller: self) do
+        consume
+      end
+
+      pause.reset
+
+      # Mark as consumed only if manual offset management is not on
+      return if topic.manual_offset_management
+
+      # We use the non-blocking one here. If someone needs the blocking one, can implement it with
+      # manual offset management
+      mark_as_consumed(messages.last)
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.consume.error'
+      )
+      client.pause(topic.name, messages.first.partition, @seek_offset || messages.first.offset)
+      pause.pause
     end
 
-    private
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_revoked
+      Karafka.monitor.instrument('consumer.revoked', caller: self) do
+        revoked
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
+    end
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+        shutdown
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
     end
 
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
@@ -53,5 +85,42 @@ module Karafka
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # Marks message as consumed in an async way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed(message)
+      client.mark_as_consumed(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Marks message as consumed in a sync way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed!(message)
+      client.mark_as_consumed!(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer] offset where we want to seek
+    def seek(offset)
+      client.seek(
+        Karafka::Messages::Seek.new(
+          messages.metadata.topic,
+          messages.metadata.partition,
+          offset
+        )
+      )
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param args [Array] All the arguments that Thor desc method accepts
-        def desc(*args)
-          @desc ||= args
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, *@desc
+          cli_class.desc name, @desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/info.rb

@@ -7,24 +7,59 @@ module Karafka
     class Info < Base
       desc 'Print configuration details and other options of your application'
 
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
+
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
-        info = [
-          "Karafka version: #{Karafka::VERSION}",
+        postfix = Karafka.pro? ? ' + Pro' : ''
+
+        [
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if Karafka.pro?
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}",
+            "License expires on: #{config.license.expires_on}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -12,19 +12,15 @@ module Karafka
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
       # @param args [Array] all the things that Thor CLI accepts
@@ -35,6 +31,7 @@ module Karafka
             Bundler.default_lockfile
           )
         ).dependencies
+
         @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
@@ -48,9 +45,7 @@ module Karafka
           target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
 
           File.open(target, 'w') { |file| file.write(render) }
         end

data/lib/karafka/cli/server.rb

@@ -5,27 +5,16 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
-      # Server config settings contract
-      CONTRACT = Contracts::ServerCliOptions.new.freeze
-
-      private_constant :CONTRACT
-
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :daemon, default: false, type: :boolean, aliases: :d
-      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
       option :consumer_groups, type: :array, default: nil, aliases: :g
 
       # Start the Karafka server
       def call
-        cli.info
-
-        validate!
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
 
-        if cli.options[:daemon]
-          FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          daemonize
-        end
+        Contracts::ServerCliOptions.new.validate!(cli.options)
 
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
@@ -36,35 +25,19 @@ module Karafka
 
       private
 
-      # Checks the server cli configuration
-      # options validations in terms of app setup (topics, pid existence, etc)
-      def validate!
-        result = CONTRACT.call(cli.options)
-        return if result.success?
-
-        raise Errors::InvalidConfigurationError, result.errors.to_h
-      end
-
-      # Detaches current process into background and writes its pidfile
-      def daemonize
-        ::Process.daemon(true)
-        File.open(
-          cli.options[:pid],
-          'w'
-        ) { |file| file.write(::Process.pid) }
-
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-      end
-
-      # Removes a pidfile (if exist)
-      def clean
-        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka.pro?
+          Karafka.logger.info(
+            "\033[0;32mThank you for investing in the Karafka Pro subscription!\033[0m\n"
+          )
+        else
+          Karafka.logger.info(
+            "\033[0;31mYou like Karafka? Please consider getting a Pro subscription!\033[0m\n"
+          )
+        end
       end
     end
   end