karafka 1.4.9 → 2.0.0.alpha1

data/karafka.gemspec

@@ -5,28 +5,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
 
-# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
-  spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
-  spec.homepage    = 'https://github.com/karafka/karafka'
+  spec.authors     = ['Maciej Mensfeld']
+  spec.email       = %w[maciej@mensfeld.pl]
+  spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.license     = 'MIT'
-
-  spec.add_dependency 'dry-configurable', '~> 0.8'
-  spec.add_dependency 'dry-inflector', '~> 0.1'
-  spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
-  spec.add_dependency 'envlogic', '~> 1.1'
-  spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'ruby-kafka', '>= 1.3.0'
+  spec.license     = 'LGPL-3.0'
+
+  spec.add_dependency 'dry-configurable', '~> 0.13'
+  spec.add_dependency 'dry-monitor', '~> 0.5'
+  spec.add_dependency 'dry-validation', '~> 1.7'
+  spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.4.0'
-  spec.add_dependency 'zeitwerk', '~> 2.1'
+  spec.add_dependency 'waterdrop', '>= 2.1.0', '< 3.0.0'
+  spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'
 
@@ -36,7 +32,11 @@ Gem::Specification.new do |spec|
 
   spec.cert_chain    = %w[certs/mensfeld.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/karafka',
+    'rubygems_mfa_required' => 'true'
+  }
 end
-# rubocop:enable Metrics/BlockLength

data/lib/active_job/consumer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
+  # It marks the offset after each message, so we make sure, none of the jobs is executed twice
+  class Consumer < Karafka::BaseConsumer
+    # Executes the ActiveJob logic
+    # @note ActiveJob does not support batches, so we just run one message after another
+    def consume
+      messages.each do |message|
+        ActiveJob::Base.execute(
+          # We technically speaking could set this as deserializer and reference it from the
+          # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+          # the ActiveJob setup here
+          ActiveSupport::JSON.decode(message.raw_payload)
+        )
+
+        mark_as_consumed(message)
+      end
+    end
+  end
+end

data/lib/active_job/karafka.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'active_job'
+require 'active_job/queue_adapters'
+require 'active_job/consumer'
+require 'active_job/routing_extensions'
+require 'active_job/queue_adapters/karafka_adapter'
+
+module ActiveJob
+  # Namespace for usage simplification outside of Rails where Railtie will not kick in.
+  # That way a require 'active_job/karafka' should be enough to use it
+  module Karafka
+  end
+end
+
+# We extend routing builder by adding a simple wrapper for easier jobs topics defining
+::Karafka::Routing::Builder.include ActiveJob::RoutingExtensions
+::Karafka::Routing::Proxy.include ActiveJob::RoutingExtensions

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# ActiveJob components to allow for jobs consumption with Karafka
+module ActiveJob
+  # ActiveJob queue adapters
+  module QueueAdapters
+    # Karafka adapter for enqueuing jobs
+    class KarafkaAdapter
+      # Enqueues the job by sending all the payload to a dedicated topic in Kafka that will be
+      # later on consumed by a special ActiveJob consumer
+      #
+      # @param job [Object] job that should be enqueued
+      def enqueue(job)
+        ::Karafka.producer.produce_async(
+          topic: job.queue_name,
+          payload: ActiveSupport::JSON.encode(job.serialize)
+        )
+      end
+
+      # Raises info, that Karafka backend does not support scheduling jobs
+      #
+      # @param _job [Object] job we cannot enqueue
+      # @param _timestamp [Time] time when job should run
+      def enqueue_at(_job, _timestamp)
+        raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
+      end
+    end
+  end
+end

data/lib/active_job/routing_extensions.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  # Routing extensions for ActiveJob
+  module RoutingExtensions
+    # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting the
+    # consumer class
+    # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+    def active_job_topic(name)
+      topic(name) do
+        consumer ActiveJob::Consumer
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,31 +6,23 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
-      end
-
-      # @return [Karafka::Routing::Builder] consumers builder instance
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
       def consumer_groups
-        config.internal.routing_builder
+        config
+          .internal
+          .routing_builder
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      def subscription_groups
+        consumer_groups
+          .active
+          .flat_map(&:subscription_groups)
       end
 
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
           App.config.internal.status.send(delegated)
@@ -42,6 +34,7 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
       ].each do |delegated|
         define_method(delegated) do

data/lib/karafka/base_consumer.rb

@@ -4,48 +4,80 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
-    extend Forwardable
-
-    # Allows us to mark messages as consumed for non-automatic mode without having
-    # to use consumer client directly. We do this that way, because most of the people should not
-    # mess with the client instance directly (just in case)
-    %i[
-      mark_as_consumed
-      mark_as_consumed!
-      trigger_heartbeat
-      trigger_heartbeat!
-    ].each do |delegated_method_name|
-      def_delegator :client, delegated_method_name
-
-      private delegated_method_name
-    end
-
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
-    attr_reader :topic
-    # @return [Karafka::Params:ParamsBatch] current params batch
-    attr_accessor :params_batch
-
-    # Assigns a topic to a consumer and builds up proper consumer functionalities
-    #   so that it can cooperate with the topic settings
-    # @param topic [Karafka::Routing::Topic]
-    def initialize(topic)
-      @topic = topic
-      Consumers::Includer.call(self)
-    end
+    attr_accessor :topic
+    # @return [Karafka::Messages::Messages] current messages batch
+    attr_accessor :messages
+    # @return [Karafka::Connection::Client] kafka connection client
+    attr_accessor :client
+    # @return [Karafka::TimeTrackers::Pause] current topic partition pause
+    attr_accessor :pause
+    # @return [Waterdrop::Producer] producer instance
+    attr_accessor :producer
 
     # Executes the default consumer flow.
-    def call
-      process
+    #
+    # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
+    #   that may not yet kick in when error occurs. That way we pause always on the last processed
+    #   message.
+    def on_consume
+      Karafka.monitor.instrument('consumer.consume', caller: self) do
+        consume
+      end
+
+      pause.reset
+
+      # Mark as consumed only if manual offset management is not on
+      return if topic.manual_offset_management
+
+      # We use the non-blocking one here. If someone needs the blocking one, can implement it with
+      # manual offset management
+      mark_as_consumed(messages.last)
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.consume.error'
+      )
+      client.pause(topic.name, messages.first.partition, @seek_offset || messages.first.offset)
+      pause.pause
     end
 
-    private
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_revoked
+      Karafka.monitor.instrument('consumer.revoked', caller: self) do
+        revoked
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.revoked.error'
+      )
+    end
 
-    # @return [Karafka::Connection::Client] messages consuming client that can be used to
-    #    commit manually offset or pause / stop consumer based on the business logic
-    def client
-      Persistence::Client.read
+    # Trigger method for running on shutdown.
+    #
+    # @private
+    def on_shutdown
+      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+        shutdown
+      end
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.shutdown.error'
+      )
     end
 
+    private
+
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
@@ -53,5 +85,42 @@ module Karafka
     def consume
       raise NotImplementedError, 'Implement this in a subclass'
     end
+
+    # Method that will be executed when a given topic partition is revoked. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def revoked; end
+
+    # Method that will be executed when the process is shutting down. You can use it for
+    # some teardown procedures (closing file handler, etc).
+    def shutdown; end
+
+    # Marks message as consumed in an async way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed(message)
+      client.mark_as_consumed(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Marks message as consumed in a sync way.
+    #
+    # @param message [Messages::Message] last successfully processed message.
+    def mark_as_consumed!(message)
+      client.mark_as_consumed!(message)
+      @seek_offset = message.offset + 1
+    end
+
+    # Seeks in the context of current topic and partition
+    #
+    # @param offset [Integer] offset where we want to seek
+    def seek(offset)
+      client.seek(
+        Karafka::Messages::Seek.new(
+          messages.metadata.topic,
+          messages.metadata.partition,
+          offset
+        )
+      )
+    end
   end
 end

data/lib/karafka/cli/base.rb

@@ -43,16 +43,16 @@ module Karafka
         end
 
         # Allows to set description of a given cli command
-        # @param args [Array] All the arguments that Thor desc method accepts
-        def desc(*args)
-          @desc ||= args
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
         end
 
         # This method will bind a given Cli command into Karafka Cli
         # This method is a wrapper to way Thor defines its commands
         # @param cli_class [Karafka::Cli] Karafka cli_class
         def bind_to(cli_class)
-          cli_class.desc name, *@desc
+          cli_class.desc name, @desc
 
           (@options || []).each { |option| cli_class.option(*option) }
 

data/lib/karafka/cli/info.rb

@@ -7,24 +7,59 @@ module Karafka
     class Info < Base
       desc 'Print configuration details and other options of your application'
 
+      # Nice karafka banner
+      BANNER = <<~BANNER
+
+        @@@                                             @@@@@  @@@
+        @@@                                            @@@     @@@
+        @@@  @@@    @@@@@@@@@   @@@ @@@   @@@@@@@@@  @@@@@@@@  @@@  @@@@   @@@@@@@@@
+        @@@@@@     @@@    @@@   @@@@@    @@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@@@@@    @@@    @@@   @@@     @@@@    @@@    @@@     @@@@@@@    @@@    @@@
+        @@@  @@@@  @@@@@@@@@@   @@@      @@@@@@@@@@    @@@     @@@  @@@@   @@@@@@@@@@
+
+      BANNER
+
       # Print configuration details and other options of your application
       def call
+        Karafka.logger.info(BANNER)
+        Karafka.logger.info((core_info + license_info).join("\n"))
+      end
+
+      private
+
+      # @return [Array<String>] core framework related info
+      def core_info
         config = Karafka::App.config
 
+        postfix = config.license.token ? ' + Pro' : ''
+
         info = [
-          "Karafka version: #{Karafka::VERSION}",
+          "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_VERSION}",
-          "Ruby-kafka version: #{::Kafka::VERSION}",
+          "Rdkafka version: #{::Rdkafka::VERSION}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
-          "Backend: #{config.backend}",
-          "Batch fetching: #{config.batch_fetching}",
-          "Batch consuming: #{config.batch_consuming}",
           "Boot file: #{Karafka.boot_file}",
-          "Environment: #{Karafka.env}",
-          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+          "Environment: #{Karafka.env}"
         ]
+      end
+
+      # @return [Array<String>] license related info
+      def license_info
+        config = Karafka::App.config
 
-        Karafka.logger.info(info.join("\n"))
+        if config.license.token
+          [
+            'License: Commercial',
+            "License entity: #{config.license.entity}",
+            "License expires on: #{config.license.expires_on}"
+          ]
+        else
+          [
+            'License: LGPL-3.0'
+          ]
+        end
       end
     end
   end

data/lib/karafka/cli/install.rb

@@ -12,19 +12,15 @@ module Karafka
       # Directories created by default
       INSTALL_DIRS = %w[
         app/consumers
-        app/responders
-        app/workers
         config
-        lib
         log
-        tmp/pids
       ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.erb' => Karafka.boot_file.basename,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
-        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+        'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
 
       # @param args [Array] all the things that Thor CLI accepts
@@ -35,6 +31,7 @@ module Karafka
             Bundler.default_lockfile
           )
         ).dependencies
+
         @rails = dependencies.key?('railties') || dependencies.key?('rails')
       end
 
@@ -48,9 +45,7 @@ module Karafka
           target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
-          # @todo Replace with the keyword argument version once we don't have to support
-          # Ruby < 2.6
-          render = ::ERB.new(template, nil, '-').result(binding)
+          render = ::ERB.new(template, trim_mode: '-').result(binding)
 
           File.open(target, 'w') { |file| file.write(render) }
         end

data/lib/karafka/cli/server.rb

@@ -12,21 +12,15 @@ module Karafka
 
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :daemon, default: false, type: :boolean, aliases: :d
-      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
       option :consumer_groups, type: :array, default: nil, aliases: :g
 
       # Start the Karafka server
       def call
-        cli.info
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
 
         validate!
 
-        if cli.options[:daemon]
-          FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          daemonize
-        end
-
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
@@ -36,6 +30,21 @@ module Karafka
 
       private
 
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka::App.config.license.token
+          Karafka.logger.info(
+            "\033[0;32mThank you for investing in the Karafka Pro subscription!\033[0m\n"
+          )
+        else
+          Karafka.logger.info(
+            "\033[0;31mYou like Karafka? Please consider getting a Pro subscription!\033[0m\n"
+          )
+        end
+      end
+
       # Checks the server cli configuration
       # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
@@ -44,28 +53,6 @@ module Karafka
 
         raise Errors::InvalidConfigurationError, result.errors.to_h
       end
-
-      # Detaches current process into background and writes its pidfile
-      def daemonize
-        ::Process.daemon(true)
-        File.open(
-          cli.options[:pid],
-          'w'
-        ) { |file| file.write(::Process.pid) }
-
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-      end
-
-      # Removes a pidfile (if exist)
-      def clean
-        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
-      end
     end
   end
 end

data/lib/karafka/cli.rb

@@ -2,6 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
+  #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
   #
@@ -10,24 +11,16 @@ module Karafka
   class Cli < Thor
     package_name 'Karafka'
 
-    default_task :missingno
-
     class << self
-      # Loads all Cli commands into Thor framework
+      # Loads all Cli commands into Thor framework.
       # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available
+      # have any Cli commands available.
       def prepare
         cli_commands.each do |action|
           action.bind_to(self)
         end
       end
 
-      # When there is a CLI crash, exit
-      # @return [true]
-      def exit_on_failure?
-        true
-      end
-
       private
 
       # @return [Array<Class>] Array with Cli action classes that can be used as commands
@@ -42,7 +35,7 @@ module Karafka
   end
 end
 
-# This is kinda trick - since we don't have a autoload and other magic stuff
+# This is kinda tricky - since we don't have an autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
 # Yes, we know that it is not turbo fast, however it is turbo convenient and small