karafka 1.3.0

data/lib/karafka/cli.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  # If you want to add/modify command that belongs to CLI, please review all commands
+  # available in cli/ directory inside Karafka source code.
+  #
+  # @note Whole Cli is built using Thor
+  # @see https://github.com/erikhuda/thor
+  class Cli < Thor
+    package_name 'Karafka'
+
+    class << self
+      # Loads all Cli commands into Thor framework
+      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
+      # have any Cli commands available
+      def prepare
+        cli_commands.each do |action|
+          action.bind_to(self)
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] Array with Cli action classes that can be used as commands
+      def cli_commands
+        constants
+          .map! { |object| const_get(object) }
+          .keep_if do |object|
+            object.instance_of?(Class) && (object < Cli::Base)
+          end
+      end
+    end
+  end
+end
+
+# This is kinda trick - since we don't have a autoload and other magic stuff
+# like Rails does, so instead this method allows us to replace currently running
+# console with a new one via Kernel.exec. It will start console with new code loaded
+# Yes, we know that it is not turbo fast, however it is turbo convenient and small
+#
+# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
+# so this method is only available when the Karafka console is running
+#
+# We skip this because this should exist and be only valid in the console
+# :nocov:
+if ENV['KARAFKA_CONSOLE']
+  # Reloads Karafka irb console session
+  def reload!
+    puts "Reloading...\n"
+    Kernel.exec Karafka::Cli::Console.command
+  end
+end
+# :nocov:

data/lib/karafka/cli/base.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Base class for all the command that we want to define
+    # This base class provides a nicer interface to Thor and allows to easier separate single
+    # independent commands
+    # In order to define a new command you need to:
+    #   - specify its desc
+    #   - implement call method
+    #
+    # @example Create a dummy command
+    #   class Dummy < Base
+    #     self.desc = 'Dummy command'
+    #
+    #     def call
+    #       puts 'I'm doing nothing!
+    #     end
+    #   end
+    class Base
+      include Thor::Shell
+
+      # We can use it to call other cli methods via this object
+      attr_reader :cli
+
+      # @param cli [Karafka::Cli] current Karafka Cli instance
+      def initialize(cli)
+        @cli = cli
+      end
+
+      # This method should implement proper cli action
+      def call
+        raise NotImplementedError, 'Implement this in a subclass'
+      end
+
+      class << self
+        # Allows to set options for Thor cli
+        # @see https://github.com/erikhuda/thor
+        # @param option Single option details
+        def option(*option)
+          @options ||= []
+          @options << option
+        end
+
+        # Allows to set description of a given cli command
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
+        end
+
+        # This method will bind a given Cli command into Karafka Cli
+        # This method is a wrapper to way Thor defines its commands
+        # @param cli_class [Karafka::Cli] Karafka cli_class
+        def bind_to(cli_class)
+          cli_class.desc name, @desc
+
+          (@options || []).each { |option| cli_class.option(*option) }
+
+          context = self
+
+          cli_class.send :define_method, name do |*args|
+            context.new(self).call(*args)
+          end
+        end
+
+        private
+
+        # @return [String] downcased current class name that we use to define name for
+        #   given Cli command
+        # @example for Karafka::Cli::Install
+        #   name #=> 'install'
+        def name
+          to_s.split('::').last.downcase
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/console.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Console Karafka Cli action
+    class Console < Base
+      desc 'Start the Karafka console (short-cut alias: "c")'
+      option aliases: 'c'
+
+      class << self
+        # @return [String] Console executing command
+        # @example
+        #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+        def command
+          envs = [
+            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+            'KARAFKA_CONSOLE=true'
+          ]
+          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        end
+      end
+
+      # Start the Karafka console
+      def call
+        cli.info
+        exec self.class.command
+      end
+    end
+  end
+end

data/lib/karafka/cli/flow.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Description of topics flow (incoming/outgoing)
+    class Flow < Base
+      desc 'Print application data flow (incoming => outgoing)'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        topics.each do |topic|
+          any_topics = !topic.responder&.topics.nil?
+
+          if any_topics
+            puts "#{topic.name} =>"
+
+            topic.responder.topics.each_value do |responder_topic|
+              features = []
+              features << (responder_topic.required? ? 'always' : 'conditionally')
+
+              print responder_topic.name, "(#{features.join(', ')})"
+            end
+          else
+            puts "#{topic.name} => (nothing)"
+          end
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
+      def topics
+        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-25s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Info Karafka Cli action
+    class Info < Base
+      desc 'Print configuration details and other options of your application'
+
+      # Print configuration details and other options of your application
+      def call
+        config = Karafka::App.config
+
+        info = [
+          "Karafka version: #{Karafka::VERSION}",
+          "Ruby version: #{RUBY_VERSION}",
+          "Ruby-kafka version: #{::Kafka::VERSION}",
+          "Application client id: #{config.client_id}",
+          "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
+          "Batch consuming: #{config.batch_consuming}",
+          "Boot file: #{Karafka.boot_file}",
+          "Environment: #{Karafka.env}",
+          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+        ]
+
+        puts(info.join("\n"))
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'erb'
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Install Karafka Cli action
+    class Install < Base
+      desc 'Install all required things for Karafka application in current directory'
+
+      # Directories created by default
+      INSTALL_DIRS = %w[
+        app/consumers
+        app/responders
+        config
+        log
+        tmp/pids
+      ].freeze
+
+      # Where should we map proper files from templates
+      INSTALL_FILES_MAP = {
+        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.erb' => 'app/responders/application_responder.rb'
+      }.freeze
+
+      # @param args [Array] all the things that Thor CLI accepts
+      def initialize(*args)
+        super
+        @rails = Bundler::LockfileParser.new(
+          Bundler.read_file(
+            Bundler.default_lockfile
+          )
+        ).dependencies.key?('rails')
+      end
+
+      # Install all required things for Karafka application in current directory
+      def call
+        INSTALL_DIRS.each do |dir|
+          FileUtils.mkdir_p Karafka.root.join(dir)
+        end
+
+        INSTALL_FILES_MAP.each do |source, target|
+          target = Karafka.root.join(target)
+
+          template = File.read(Karafka.core_root.join("templates/#{source}"))
+          # @todo Replace with the keyword argument version once we don't have to support
+          # Ruby < 2.6
+          render = ::ERB.new(template, nil, '-').result(binding)
+
+          File.open(target, 'w') { |file| file.write(render) }
+        end
+      end
+
+      # @return [Boolean] true if we have Rails loaded
+      # This allows us to generate customized karafka.rb template with some tweaks specific for
+      # Rails
+      def rails?
+        @rails
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Server Karafka Cli action
+    class Server < Base
+      # Server config settings contract
+      CONTRACT = Contracts::ServerCliOptions.new.freeze
+
+      private_constant :CONTRACT
+
+      desc 'Start the Karafka server (short-cut alias: "s")'
+      option aliases: 's'
+      option :daemon, default: false, type: :boolean, aliases: :d
+      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+      option :consumer_groups, type: :array, default: nil, aliases: :g
+
+      # Start the Karafka server
+      def call
+        cli.info
+
+        validate!
+
+        if cli.options[:daemon]
+          FileUtils.mkdir_p File.dirname(cli.options[:pid])
+          daemonize
+        end
+
+        # We assign active topics on a server level, as only server is expected to listen on
+        # part of the topics
+        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
+        Karafka::Server.run
+      end
+
+      private
+
+      # Checks the server cli configuration
+      # options validations in terms of app setup (topics, pid existence, etc)
+      def validate!
+        result = CONTRACT.call(cli.options)
+        return if result.success?
+
+        raise Errors::InvalidConfigurationError, result.errors.to_h
+      end
+
+      # Detaches current process into background and writes its pidfile
+      def daemonize
+        ::Process.daemon(true)
+        File.open(
+          cli.options[:pid],
+          'w'
+        ) { |file| file.write(::Process.pid) }
+      end
+
+      # Removes a pidfile (if exist)
+      def clean
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      end
+    end
+  end
+end

data/lib/karafka/code_reloader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Special type of a listener, that is not an instrumentation one, but one that triggers
+  # code reload in the development mode after each fetched batch (or message)
+  #
+  # Please refer to the development code reload sections for details on the benefits and downsides
+  # of the in-process code reloading
+  class CodeReloader
+    # This mutex is needed as we might have an application that has multiple consumer groups
+    # running in separate threads and we should not trigger reload before fully reloading the app
+    # in previous thread
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
+
+    # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
+    #  the Rails loader, Zeitwerk or anything else that allows reloading triggering
+    # @param block [Proc] yields given block just before reloading. This can be used to hook custom
+    #   reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
+    def initialize(*reloaders, &block)
+      @reloaders = reloaders
+      @block = block
+    end
+
+    # Binds to the instrumentation events and triggers reload
+    # @param _event [Dry::Event] empty dry event
+    # @note Since we de-register all the user defined objects and redraw routes, it means that
+    #   we won't be able to do a multi-batch buffering in the development mode as each of the
+    #   batches will be buffered on a newly created "per fetch" instance.
+    def on_connection_listener_fetch_loop(_event)
+      reload
+    end
+
+    private
+
+    # Triggers reload of both standard and Rails reloaders as well as expires all internals of
+    # Karafka, so it can be rediscovered and rebuilt
+    def reload
+      MUTEX.synchronize do
+        if @reloaders[0].respond_to?(:execute)
+          reload_with_rails
+        else
+          reload_without_rails
+        end
+      end
+    end
+
+    # Rails reloading procedure
+    def reload_with_rails
+      updatable = @reloaders.select(&:updated?)
+
+      return if updatable.empty?
+
+      updatable.each(&:execute)
+      @block&.call
+      Karafka::App.reload
+    end
+
+    # Zeitwerk and other reloaders
+    def reload_without_rails
+      @reloaders.each(&:reload)
+      @block&.call
+      Karafka::App.reload
+    end
+  end
+end