karafka 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +68 -0
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +6 -0
  6. data/CHANGELOG.md +202 -0
  7. data/Gemfile +8 -0
  8. data/Gemfile.lock +216 -0
  9. data/MIT-LICENCE +18 -0
  10. data/README.md +831 -0
  11. data/Rakefile +17 -0
  12. data/bin/karafka +7 -0
  13. data/karafka.gemspec +34 -0
  14. data/lib/karafka.rb +73 -0
  15. data/lib/karafka/app.rb +45 -0
  16. data/lib/karafka/base_controller.rb +162 -0
  17. data/lib/karafka/base_responder.rb +118 -0
  18. data/lib/karafka/base_worker.rb +41 -0
  19. data/lib/karafka/capistrano.rb +2 -0
  20. data/lib/karafka/capistrano/karafka.cap +84 -0
  21. data/lib/karafka/cli.rb +52 -0
  22. data/lib/karafka/cli/base.rb +74 -0
  23. data/lib/karafka/cli/console.rb +23 -0
  24. data/lib/karafka/cli/flow.rb +46 -0
  25. data/lib/karafka/cli/info.rb +26 -0
  26. data/lib/karafka/cli/install.rb +45 -0
  27. data/lib/karafka/cli/routes.rb +39 -0
  28. data/lib/karafka/cli/server.rb +59 -0
  29. data/lib/karafka/cli/worker.rb +26 -0
  30. data/lib/karafka/connection/consumer.rb +29 -0
  31. data/lib/karafka/connection/listener.rb +54 -0
  32. data/lib/karafka/connection/message.rb +17 -0
  33. data/lib/karafka/connection/topic_consumer.rb +48 -0
  34. data/lib/karafka/errors.rb +50 -0
  35. data/lib/karafka/fetcher.rb +40 -0
  36. data/lib/karafka/helpers/class_matcher.rb +77 -0
  37. data/lib/karafka/helpers/multi_delegator.rb +31 -0
  38. data/lib/karafka/loader.rb +77 -0
  39. data/lib/karafka/logger.rb +52 -0
  40. data/lib/karafka/monitor.rb +82 -0
  41. data/lib/karafka/params/interchanger.rb +33 -0
  42. data/lib/karafka/params/params.rb +102 -0
  43. data/lib/karafka/patches/dry/configurable/config.rb +37 -0
  44. data/lib/karafka/process.rb +61 -0
  45. data/lib/karafka/responders/builder.rb +33 -0
  46. data/lib/karafka/responders/topic.rb +43 -0
  47. data/lib/karafka/responders/usage_validator.rb +59 -0
  48. data/lib/karafka/routing/builder.rb +89 -0
  49. data/lib/karafka/routing/route.rb +80 -0
  50. data/lib/karafka/routing/router.rb +38 -0
  51. data/lib/karafka/server.rb +53 -0
  52. data/lib/karafka/setup/config.rb +57 -0
  53. data/lib/karafka/setup/configurators/base.rb +33 -0
  54. data/lib/karafka/setup/configurators/celluloid.rb +20 -0
  55. data/lib/karafka/setup/configurators/sidekiq.rb +34 -0
  56. data/lib/karafka/setup/configurators/water_drop.rb +19 -0
  57. data/lib/karafka/setup/configurators/worker_glass.rb +13 -0
  58. data/lib/karafka/status.rb +23 -0
  59. data/lib/karafka/templates/app.rb.example +26 -0
  60. data/lib/karafka/templates/application_controller.rb.example +5 -0
  61. data/lib/karafka/templates/application_responder.rb.example +9 -0
  62. data/lib/karafka/templates/application_worker.rb.example +12 -0
  63. data/lib/karafka/templates/config.ru.example +13 -0
  64. data/lib/karafka/templates/sidekiq.yml.example +26 -0
  65. data/lib/karafka/version.rb +6 -0
  66. data/lib/karafka/workers/builder.rb +49 -0
  67. data/log/.gitkeep +0 -0
  68. metadata +267 -0
@@ -0,0 +1,41 @@
1
+ module Karafka
2
+ # Worker wrapper for Sidekiq workers
3
+ class BaseWorker
4
+ include Sidekiq::Worker
5
+
6
+ attr_accessor :params, :topic
7
+
8
+ # Executes the logic that lies in #perform Karafka controller method
9
+ # @param topic [String] Topic that we will use to route to a proper controller
10
+ # @param params [Hash] params hash that we use to build Karafka params object
11
+ def perform(topic, params)
12
+ self.topic = topic
13
+ self.params = params
14
+ Karafka.monitor.notice(self.class, controller.to_h)
15
+ controller.perform
16
+ end
17
+
18
+ # What action should be taken when perform method fails
19
+ # @param topic [String] Topic bthat we will use to route to a proper controller
20
+ # @param params [Hash] params hash that we use to build Karafka params object
21
+ def after_failure(topic, params)
22
+ self.topic = topic
23
+ self.params = params
24
+
25
+ return unless controller.respond_to?(:after_failure)
26
+
27
+ Karafka.monitor.notice(self.class, controller.to_h)
28
+ controller.after_failure
29
+ end
30
+
31
+ private
32
+
33
+ # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
34
+ # with params assigned already (controller is ready to use)
35
+ def controller
36
+ @controller ||= Karafka::Routing::Router.new(topic).build.tap do |ctrl|
37
+ ctrl.params = ctrl.interchanger.parse(params)
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,2 @@
1
+ # Load Capistrano tasks only if Capistrano is loaded
2
+ load File.expand_path('../capistrano/karafka.cap', __FILE__) if defined?(Capistrano)
@@ -0,0 +1,84 @@
1
+ # @note Inspired by Puma capistrano handlers
2
+ # @see https://github.com/seuros/capistrano-puma/blob/master/lib/capistrano/tasks/puma.rake
3
+ namespace :load do
4
+ task :defaults do
5
+ set :karafka_default_hooks, -> { true }
6
+ set :karafka_env, -> { fetch(:karafka_env, fetch(:environment)) }
7
+ set :karafka_pid, -> { File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
8
+ end
9
+ end
10
+
11
+ namespace :deploy do
12
+ before :starting, :check_karafka_hooks do
13
+ invoke 'karafka:add_default_hooks' if fetch(:karafka_default_hooks)
14
+ end
15
+ end
16
+
17
+ namespace :karafka do
18
+ desc 'Stop Karafka'
19
+ task :stop do
20
+ on roles(:app) do |host|
21
+ within shared_path do
22
+ # If there's no pidfile it means that Karafka is not running
23
+ next unless test "cat #{fetch(:karafka_pid)}"
24
+
25
+ # Send a kill signal to a given process
26
+ execute "kill -INT `cat #{fetch(:karafka_pid)}`"
27
+
28
+ # And wait until it finishes. We wait because we don't want to start next process until
29
+ # the previous one is stopped. That way we won't have problems with Kafka registering and
30
+ # deregistering processes from topics (although nothing bad would happen. It would just
31
+ # take more time to rebalance)
32
+ while true
33
+ break unless test "cat #{fetch(:karafka_pid)}"
34
+ info 'Waiting for Karafka to stop'
35
+ sleep 5
36
+ end
37
+ end
38
+ end
39
+ end
40
+
41
+ desc 'Start Karafka'
42
+ task :start do
43
+ on roles(:app) do |host|
44
+ within current_path do
45
+ # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
46
+ # so we want to set proper env for any of them
47
+ with(
48
+ KARAFKA_ENV: fetch(:karafka_env),
49
+ RAILS_ENV: fetch(:rails_env),
50
+ RACK_ENV: fetch(:rack_env)
51
+ )do
52
+ execute :bundle, "exec karafka server -d -p #{fetch(:karafka_pid)}"
53
+ end
54
+ end
55
+ end
56
+ end
57
+
58
+ desc 'Restart Karafka'
59
+ task :restart do
60
+ invoke 'karafka:stop'
61
+ invoke 'karafka:start'
62
+ end
63
+
64
+ desc 'Status Karafka'
65
+ task :status do
66
+ on roles(:app) do |host|
67
+ if test "cat #{fetch(:karafka_pid)}"
68
+ pid = capture "cat #{fetch(:karafka_pid)}"
69
+
70
+ if test "ps -p #{pid} > /dev/null"
71
+ info "Karafka is started: #{pid}"
72
+ else
73
+ error "Karafka is not started but pidfile exists"
74
+ end
75
+ else
76
+ info "Karafka is not started"
77
+ end
78
+ end
79
+ end
80
+
81
+ task :add_default_hooks do
82
+ after 'deploy:finished', 'karafka:restart'
83
+ end
84
+ end
@@ -0,0 +1,52 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ # If you want to add/modify command that belongs to CLI, please review all commands
4
+ # available in cli/ directory inside Karafka source code.
5
+ #
6
+ # @note Whole Cli is built using Thor
7
+ # @see https://github.com/erikhuda/thor
8
+ class Cli
9
+ package_name 'Karafka'
10
+
11
+ class << self
12
+ # Loads all Cli commands into Thor framework
13
+ # This method should be executed before we run Karafka::Cli.start, otherwise we won't
14
+ # have any Cli commands available
15
+ def prepare
16
+ cli_commands.each do |action|
17
+ action.bind_to(self)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ # @return [Array<Class>] Array with Cli action classes that can be used as commands
24
+ def cli_commands
25
+ constants
26
+ .map! { |object| const_get(object) }
27
+ .keep_if do |object|
28
+ object.instance_of?(Class) && (object < Cli::Base)
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ # This is kinda trick - since we don't have a autoload and other magic stuff
36
+ # like Rails does, so instead this method allows us to replace currently running
37
+ # console with a new one via Kernel.exec. It will start console with new code loaded
38
+ # Yes we know that it is not turbofast, however it is turbo convinient and small
39
+ #
40
+ # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
41
+ # so this method is only available when the Karafka console is running
42
+ #
43
+ # We skip this because this should exist and be only valid in the console
44
+ # :nocov:
45
+ if ENV['KARAFKA_CONSOLE']
46
+ # Reloads Karafka irb console session
47
+ def reload!
48
+ puts "Reloading...\n"
49
+ Kernel.exec Karafka::Cli::Console.command
50
+ end
51
+ end
52
+ # :nocov:
@@ -0,0 +1,74 @@
1
+ module Karafka
2
+ class Cli < Thor
3
+ # Base class for all the command that we want to define
4
+ # This base class provides a nicer interface to Thor and allows to easier separate single
5
+ # independent commands
6
+ # In order to define a new command you need to:
7
+ # - specify its desc
8
+ # - implement call method
9
+ #
10
+ # @example Create a dummy command
11
+ # class Dummy < Base
12
+ # self.desc = 'Dummy command'
13
+ #
14
+ # def call
15
+ # puts 'I'm doing nothing!
16
+ # end
17
+ # end
18
+ class Base
19
+ # We can use it to call other cli methods via this object
20
+ attr_reader :cli
21
+
22
+ # @param cli [Karafka::Cli] current Karafka Cli instance
23
+ def initialize(cli)
24
+ @cli = cli
25
+ end
26
+
27
+ # This method should implement proper cli action
28
+ def call
29
+ raise NotImplementedError, 'Implement this in a subclass'
30
+ end
31
+
32
+ class << self
33
+ # Allows to set options for Thor cli
34
+ # @see https://github.com/erikhuda/thor
35
+ # @param option Single option details
36
+ def option(*option)
37
+ @options ||= []
38
+ @options << option
39
+ end
40
+
41
+ # Allows to set description of a given cli command
42
+ # @param desc [String] Description of a given cli command
43
+ def desc(desc)
44
+ @desc ||= desc
45
+ end
46
+
47
+ # This method will bind a given Cli command into Karafka Cli
48
+ # This method is a wrapper to way Thor defines its commands
49
+ # @param cli_class [Karafka::Cli] Karafka cli_class
50
+ def bind_to(cli_class)
51
+ cli_class.desc name, @desc
52
+
53
+ (@options || []).each { |option| cli_class.option(*option) }
54
+
55
+ context = self
56
+
57
+ cli_class.send :define_method, name do |*args|
58
+ context.new(self).call(*args)
59
+ end
60
+ end
61
+
62
+ private
63
+
64
+ # @return [String] downcased current class name that we use to define name for
65
+ # given Cli command
66
+ # @example for Karafka::Cli::Install
67
+ # name #=> 'install'
68
+ def name
69
+ to_s.split('::').last.downcase
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,23 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Console Karafka Cli action
5
+ class Console < Base
6
+ desc 'Start the Karafka console (short-cut alias: "c")'
7
+ option aliases: 'c'
8
+
9
+ # @return [String] Console executing command
10
+ # @example
11
+ # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
12
+ def self.command
13
+ "KARAFKA_CONSOLE=true bundle exec irb -r #{Karafka.boot_file}"
14
+ end
15
+
16
+ # Start the Karafka console
17
+ def call
18
+ cli.info
19
+ system self.class.command
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,46 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Description of topics flow (incoming/outgoing)
5
+ class Flow < Base
6
+ desc 'Print application data flow (incoming => outgoing)'
7
+
8
+ # Print out all defined routes in alphabetical order
9
+ def call
10
+ routes.each do |route|
11
+ any_topics = !route.responder&.topics.nil?
12
+
13
+ if any_topics
14
+ puts "#{route.topic} =>"
15
+
16
+ route.responder.topics.each do |_name, topic|
17
+ features = []
18
+ features << (topic.required? ? 'always' : 'conditionally')
19
+ features << (topic.multiple_usage? ? 'one or more' : 'exactly once')
20
+
21
+ print topic.name, "(#{features.join(', ')})"
22
+ end
23
+ else
24
+ puts "#{route.topic} => (nothing)"
25
+ end
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
32
+ def routes
33
+ Karafka::App.routes.sort do |route1, route2|
34
+ route1.topic <=> route2.topic
35
+ end
36
+ end
37
+
38
+ # Prints a given value with label in a nice way
39
+ # @param label [String] label describing value
40
+ # @param value [String] value that should be printed
41
+ def print(label, value)
42
+ printf "%-25s %s\n", " - #{label}:", value
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,26 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Info Karafka Cli action
5
+ class Info < Base
6
+ desc 'Print configuration details and other options of your application'
7
+
8
+ # Print configuration details and other options of your application
9
+ def call
10
+ config = Karafka::App.config
11
+
12
+ info = [
13
+ "Karafka framework version: #{Karafka::VERSION}",
14
+ "Application name: #{config.name}",
15
+ "Number of threads: #{config.concurrency}",
16
+ "Boot file: #{Karafka.boot_file}",
17
+ "Environment: #{Karafka.env}",
18
+ "Kafka hosts: #{config.kafka.hosts}",
19
+ "Redis: #{config.redis.to_h}"
20
+ ]
21
+
22
+ puts(info.join("\n"))
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,45 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Install Karafka Cli action
5
+ class Install < Base
6
+ desc 'Install all required things for Karafka application in current directory'
7
+
8
+ # Directories created by default
9
+ INSTALL_DIRS = %w(
10
+ app/models
11
+ app/controllers
12
+ app/responders
13
+ app/workers
14
+ config
15
+ log
16
+ tmp/pids
17
+ ).freeze
18
+
19
+ # Where should we map proper files from templates
20
+ INSTALL_FILES_MAP = {
21
+ 'app.rb.example' => Karafka.boot_file.basename,
22
+ 'config.ru.example' => 'config.ru',
23
+ 'sidekiq.yml.example' => 'config/sidekiq.yml.example',
24
+ 'application_worker.rb.example' => 'app/workers/application_worker.rb',
25
+ 'application_controller.rb.example' => 'app/controllers/application_controller.rb',
26
+ 'application_responder.rb.example' => 'app/responders/application_responder.rb'
27
+ }.freeze
28
+
29
+ # Install all required things for Karafka application in current directory
30
+ def call
31
+ INSTALL_DIRS.each do |dir|
32
+ FileUtils.mkdir_p Karafka.root.join(dir)
33
+ end
34
+
35
+ INSTALL_FILES_MAP.each do |source, target|
36
+ target = Karafka.root.join(target)
37
+ next if File.exist?(target)
38
+
39
+ source = Karafka.core_root.join("templates/#{source}")
40
+ FileUtils.cp_r(source, target)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,39 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Routes Karafka Cli action
5
+ class Routes < Base
6
+ desc 'Print out all defined routes in alphabetical order'
7
+ option aliases: 'r'
8
+
9
+ # Print out all defined routes in alphabetical order
10
+ def call
11
+ routes.each do |route|
12
+ puts "#{route.topic}:"
13
+ print('Group', route.group)
14
+ print('Controller', route.controller)
15
+ print('Worker', route.worker)
16
+ print('Parser', route.parser)
17
+ print('Interchanger', route.interchanger)
18
+ print('Responder', route.responder)
19
+ end
20
+ end
21
+
22
+ private
23
+
24
+ # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
25
+ def routes
26
+ Karafka::App.routes.sort do |route1, route2|
27
+ route1.topic <=> route2.topic
28
+ end
29
+ end
30
+
31
+ # Prints a given value with label in a nice way
32
+ # @param label [String] label describing value
33
+ # @param value [String] value that should be printed
34
+ def print(label, value)
35
+ printf "%-18s %s\n", " - #{label}:", value
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,59 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Server Karafka Cli action
5
+ class Server < Base
6
+ desc 'Start the Karafka server (short-cut alias: "s")'
7
+ option aliases: 's'
8
+ option :daemon, default: false, type: :boolean, aliases: :d
9
+ option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
10
+
11
+ # Start the Karafka server
12
+ def call
13
+ puts 'Starting Karafka server'
14
+ cli.info
15
+
16
+ if cli.options[:daemon]
17
+ # For some reason Celluloid spins threads that break forking
18
+ # Threads are not shutdown immediately so deamonization will stale until
19
+ # those threads are killed by Celluloid manager (via timeout)
20
+ # There's nothing initialized here yet, so instead we shutdown celluloid
21
+ # and run it again when we need (after fork)
22
+ Celluloid.shutdown
23
+ validate!
24
+ daemonize
25
+ Celluloid.boot
26
+ end
27
+
28
+ # Remove pidfile on shutdown
29
+ ObjectSpace.define_finalizer('string', proc { send(:clean) })
30
+
31
+ # After we fork, we can boot celluloid again
32
+ Karafka::Server.run
33
+ end
34
+
35
+ private
36
+
37
+ # Prepare (if not exists) directory for a pidfile and check if there is no running karafka
38
+ # instance already (and raise error if so)
39
+ def validate!
40
+ FileUtils.mkdir_p File.dirname(cli.options[:pid])
41
+ raise "#{cli.options[:pid]} already exists" if File.exist?(cli.options[:pid])
42
+ end
43
+
44
+ # Detaches current process into background and writes its pidfile
45
+ def daemonize
46
+ ::Process.daemon(true)
47
+ File.open(
48
+ cli.options[:pid],
49
+ 'w'
50
+ ) { |file| file.write(::Process.pid) }
51
+ end
52
+
53
+ # Removes a pidfile (if exist)
54
+ def clean
55
+ FileUtils.rm_f(cli.options[:pid])
56
+ end
57
+ end
58
+ end
59
+ end