karafka 0.5.0

data/lib/karafka/base_worker.rb

@@ -0,0 +1,41 @@
+module Karafka
+  # Worker wrapper for Sidekiq workers
+  class BaseWorker
+    include Sidekiq::Worker
+
+    attr_accessor :params, :topic
+
+    # Executes the logic that lies in #perform Karafka controller method
+    # @param topic [String] Topic that we will use to route to a proper controller
+    # @param params [Hash] params hash that we use to build Karafka params object
+    def perform(topic, params)
+      self.topic = topic
+      self.params = params
+      Karafka.monitor.notice(self.class, controller.to_h)
+      controller.perform
+    end
+
+    # What action should be taken when perform method fails
+    # @param topic [String] Topic bthat we will use to route to a proper controller
+    # @param params [Hash] params hash that we use to build Karafka params object
+    def after_failure(topic, params)
+      self.topic = topic
+      self.params = params
+
+      return unless controller.respond_to?(:after_failure)
+
+      Karafka.monitor.notice(self.class, controller.to_h)
+      controller.after_failure
+    end
+
+    private
+
+    # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
+    #   with params assigned already (controller is ready to use)
+    def controller
+      @controller ||= Karafka::Routing::Router.new(topic).build.tap do |ctrl|
+        ctrl.params = ctrl.interchanger.parse(params)
+      end
+    end
+  end
+end

data/lib/karafka/capistrano.rb

@@ -0,0 +1,2 @@
+# Load Capistrano tasks only if Capistrano is loaded
+load File.expand_path('../capistrano/karafka.cap', __FILE__) if defined?(Capistrano)

data/lib/karafka/capistrano/karafka.cap

@@ -0,0 +1,84 @@
+# @note Inspired by Puma capistrano handlers
+# @see https://github.com/seuros/capistrano-puma/blob/master/lib/capistrano/tasks/puma.rake
+namespace :load do
+  task :defaults do
+    set :karafka_default_hooks, -> { true }
+    set :karafka_env, -> { fetch(:karafka_env, fetch(:environment)) }
+    set :karafka_pid, -> { File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
+  end
+end
+
+namespace :deploy do
+  before :starting, :check_karafka_hooks do
+    invoke 'karafka:add_default_hooks' if fetch(:karafka_default_hooks)
+  end
+end
+
+namespace :karafka do
+  desc 'Stop Karafka'
+  task :stop do
+    on roles(:app) do |host|
+      within shared_path do
+        # If there's no pidfile it means that Karafka is not running
+        next unless test "cat #{fetch(:karafka_pid)}"
+
+        # Send a kill signal to a given process
+        execute "kill -INT `cat #{fetch(:karafka_pid)}`"
+
+        # And wait until it finishes. We wait because we don't want to start next process until
+        # the previous one is stopped. That way we won't have problems with Kafka registering and
+        # deregistering processes from topics (although nothing bad would happen. It would just
+        # take more time to rebalance)
+        while true
+          break unless test "cat #{fetch(:karafka_pid)}"
+          info 'Waiting for Karafka to stop'
+          sleep 5
+        end
+      end
+    end
+  end
+
+  desc 'Start Karafka'
+  task :start do
+    on roles(:app) do |host|
+      within current_path do
+        # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
+        # so we want to set proper env for any of them
+        with(
+          KARAFKA_ENV: fetch(:karafka_env),
+          RAILS_ENV: fetch(:rails_env),
+          RACK_ENV: fetch(:rack_env)
+        )do
+          execute :bundle, "exec karafka server -d -p #{fetch(:karafka_pid)}"
+        end
+      end
+    end
+  end
+
+  desc 'Restart Karafka'
+  task :restart do
+    invoke 'karafka:stop'
+    invoke 'karafka:start'
+  end
+
+  desc 'Status Karafka'
+  task :status do
+    on roles(:app) do |host|
+      if test "cat #{fetch(:karafka_pid)}"
+        pid = capture "cat #{fetch(:karafka_pid)}"
+
+        if test "ps -p #{pid} > /dev/null"
+          info "Karafka is started: #{pid}"
+        else
+          error "Karafka is not started but pidfile exists"
+        end
+      else
+        info "Karafka is not started"
+      end
+    end
+  end
+
+  task :add_default_hooks do
+    after 'deploy:finished', 'karafka:restart'
+  end
+end

data/lib/karafka/cli.rb

@@ -0,0 +1,52 @@
+module Karafka
+  # Karafka framework Cli
+  # If you want to add/modify command that belongs to CLI, please review all commands
+  # available in cli/ directory inside Karafka source code.
+  #
+  # @note Whole Cli is built using Thor
+  # @see https://github.com/erikhuda/thor
+  class Cli
+    package_name 'Karafka'
+
+    class << self
+      # Loads all Cli commands into Thor framework
+      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
+      # have any Cli commands available
+      def prepare
+        cli_commands.each do |action|
+          action.bind_to(self)
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] Array with Cli action classes that can be used as commands
+      def cli_commands
+        constants
+          .map! { |object| const_get(object) }
+          .keep_if do |object|
+            object.instance_of?(Class) && (object < Cli::Base)
+          end
+      end
+    end
+  end
+end
+
+# This is kinda trick - since we don't have a autoload and other magic stuff
+# like Rails does, so instead this method allows us to replace currently running
+# console with a new one via Kernel.exec. It will start console with new code loaded
+# Yes we know that it is not turbofast, however it is turbo convinient and small
+#
+# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
+# so this method is only available when the Karafka console is running
+#
+# We skip this because this should exist and be only valid in the console
+# :nocov:
+if ENV['KARAFKA_CONSOLE']
+  # Reloads Karafka irb console session
+  def reload!
+    puts "Reloading...\n"
+    Kernel.exec Karafka::Cli::Console.command
+  end
+end
+# :nocov:

data/lib/karafka/cli/base.rb

@@ -0,0 +1,74 @@
+module Karafka
+  class Cli < Thor
+    # Base class for all the command that we want to define
+    # This base class provides a nicer interface to Thor and allows to easier separate single
+    # independent commands
+    # In order to define a new command you need to:
+    #   - specify its desc
+    #   - implement call method
+    #
+    # @example Create a dummy command
+    #   class Dummy < Base
+    #     self.desc = 'Dummy command'
+    #
+    #     def call
+    #       puts 'I'm doing nothing!
+    #     end
+    #   end
+    class Base
+      # We can use it to call other cli methods via this object
+      attr_reader :cli
+
+      # @param cli [Karafka::Cli] current Karafka Cli instance
+      def initialize(cli)
+        @cli = cli
+      end
+
+      # This method should implement proper cli action
+      def call
+        raise NotImplementedError, 'Implement this in a subclass'
+      end
+
+      class << self
+        # Allows to set options for Thor cli
+        # @see https://github.com/erikhuda/thor
+        # @param option Single option details
+        def option(*option)
+          @options ||= []
+          @options << option
+        end
+
+        # Allows to set description of a given cli command
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
+        end
+
+        # This method will bind a given Cli command into Karafka Cli
+        # This method is a wrapper to way Thor defines its commands
+        # @param cli_class [Karafka::Cli] Karafka cli_class
+        def bind_to(cli_class)
+          cli_class.desc name, @desc
+
+          (@options || []).each { |option| cli_class.option(*option) }
+
+          context = self
+
+          cli_class.send :define_method, name do |*args|
+            context.new(self).call(*args)
+          end
+        end
+
+        private
+
+        # @return [String] downcased current class name that we use to define name for
+        #   given Cli command
+        # @example for Karafka::Cli::Install
+        #   name #=> 'install'
+        def name
+          to_s.split('::').last.downcase
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/console.rb

@@ -0,0 +1,23 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Console Karafka Cli action
+    class Console < Base
+      desc 'Start the Karafka console (short-cut alias: "c")'
+      option aliases: 'c'
+
+      # @return [String] Console executing command
+      # @example
+      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+      def self.command
+        "KARAFKA_CONSOLE=true bundle exec irb -r #{Karafka.boot_file}"
+      end
+
+      # Start the Karafka console
+      def call
+        cli.info
+        system self.class.command
+      end
+    end
+  end
+end

data/lib/karafka/cli/flow.rb

@@ -0,0 +1,46 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Description of topics flow (incoming/outgoing)
+    class Flow < Base
+      desc 'Print application data flow (incoming => outgoing)'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        routes.each do |route|
+          any_topics = !route.responder&.topics.nil?
+
+          if any_topics
+            puts "#{route.topic} =>"
+
+            route.responder.topics.each do |_name, topic|
+              features = []
+              features << (topic.required? ? 'always' : 'conditionally')
+              features << (topic.multiple_usage? ? 'one or more' : 'exactly once')
+
+              print topic.name, "(#{features.join(', ')})"
+            end
+          else
+            puts "#{route.topic} => (nothing)"
+          end
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
+      def routes
+        Karafka::App.routes.sort do |route1, route2|
+          route1.topic <=> route2.topic
+        end
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-25s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -0,0 +1,26 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Info Karafka Cli action
+    class Info < Base
+      desc 'Print configuration details and other options of your application'
+
+      # Print configuration details and other options of your application
+      def call
+        config = Karafka::App.config
+
+        info = [
+          "Karafka framework version: #{Karafka::VERSION}",
+          "Application name: #{config.name}",
+          "Number of threads: #{config.concurrency}",
+          "Boot file: #{Karafka.boot_file}",
+          "Environment: #{Karafka.env}",
+          "Kafka hosts: #{config.kafka.hosts}",
+          "Redis: #{config.redis.to_h}"
+        ]
+
+        puts(info.join("\n"))
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -0,0 +1,45 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Install Karafka Cli action
+    class Install < Base
+      desc 'Install all required things for Karafka application in current directory'
+
+      # Directories created by default
+      INSTALL_DIRS = %w(
+        app/models
+        app/controllers
+        app/responders
+        app/workers
+        config
+        log
+        tmp/pids
+      ).freeze
+
+      # Where should we map proper files from templates
+      INSTALL_FILES_MAP = {
+        'app.rb.example' => Karafka.boot_file.basename,
+        'config.ru.example' => 'config.ru',
+        'sidekiq.yml.example' => 'config/sidekiq.yml.example',
+        'application_worker.rb.example' => 'app/workers/application_worker.rb',
+        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
+        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+      }.freeze
+
+      # Install all required things for Karafka application in current directory
+      def call
+        INSTALL_DIRS.each do |dir|
+          FileUtils.mkdir_p Karafka.root.join(dir)
+        end
+
+        INSTALL_FILES_MAP.each do |source, target|
+          target = Karafka.root.join(target)
+          next if File.exist?(target)
+
+          source = Karafka.core_root.join("templates/#{source}")
+          FileUtils.cp_r(source, target)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/routes.rb

@@ -0,0 +1,39 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Routes Karafka Cli action
+    class Routes < Base
+      desc 'Print out all defined routes in alphabetical order'
+      option aliases: 'r'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        routes.each do |route|
+          puts "#{route.topic}:"
+          print('Group', route.group)
+          print('Controller', route.controller)
+          print('Worker', route.worker)
+          print('Parser', route.parser)
+          print('Interchanger', route.interchanger)
+          print('Responder', route.responder)
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
+      def routes
+        Karafka::App.routes.sort do |route1, route2|
+          route1.topic <=> route2.topic
+        end
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-18s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -0,0 +1,59 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Server Karafka Cli action
+    class Server < Base
+      desc 'Start the Karafka server (short-cut alias: "s")'
+      option aliases: 's'
+      option :daemon, default: false, type: :boolean, aliases: :d
+      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+
+      # Start the Karafka server
+      def call
+        puts 'Starting Karafka server'
+        cli.info
+
+        if cli.options[:daemon]
+          # For some reason Celluloid spins threads that break forking
+          # Threads are not shutdown immediately so deamonization will stale until
+          # those threads are killed by Celluloid manager (via timeout)
+          # There's nothing initialized here yet, so instead we shutdown celluloid
+          # and run it again when we need (after fork)
+          Celluloid.shutdown
+          validate!
+          daemonize
+          Celluloid.boot
+        end
+
+        # Remove pidfile on shutdown
+        ObjectSpace.define_finalizer('string', proc { send(:clean) })
+
+        # After we fork, we can boot celluloid again
+        Karafka::Server.run
+      end
+
+      private
+
+      # Prepare (if not exists) directory for a pidfile and check if there is no running karafka
+      # instance already (and raise error if so)
+      def validate!
+        FileUtils.mkdir_p File.dirname(cli.options[:pid])
+        raise "#{cli.options[:pid]} already exists" if File.exist?(cli.options[:pid])
+      end
+
+      # Detaches current process into background and writes its pidfile
+      def daemonize
+        ::Process.daemon(true)
+        File.open(
+          cli.options[:pid],
+          'w'
+        ) { |file| file.write(::Process.pid) }
+      end
+
+      # Removes a pidfile (if exist)
+      def clean
+        FileUtils.rm_f(cli.options[:pid])
+      end
+    end
+  end
+end