karafka 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +68 -0
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +6 -0
  6. data/CHANGELOG.md +202 -0
  7. data/Gemfile +8 -0
  8. data/Gemfile.lock +216 -0
  9. data/MIT-LICENCE +18 -0
  10. data/README.md +831 -0
  11. data/Rakefile +17 -0
  12. data/bin/karafka +7 -0
  13. data/karafka.gemspec +34 -0
  14. data/lib/karafka.rb +73 -0
  15. data/lib/karafka/app.rb +45 -0
  16. data/lib/karafka/base_controller.rb +162 -0
  17. data/lib/karafka/base_responder.rb +118 -0
  18. data/lib/karafka/base_worker.rb +41 -0
  19. data/lib/karafka/capistrano.rb +2 -0
  20. data/lib/karafka/capistrano/karafka.cap +84 -0
  21. data/lib/karafka/cli.rb +52 -0
  22. data/lib/karafka/cli/base.rb +74 -0
  23. data/lib/karafka/cli/console.rb +23 -0
  24. data/lib/karafka/cli/flow.rb +46 -0
  25. data/lib/karafka/cli/info.rb +26 -0
  26. data/lib/karafka/cli/install.rb +45 -0
  27. data/lib/karafka/cli/routes.rb +39 -0
  28. data/lib/karafka/cli/server.rb +59 -0
  29. data/lib/karafka/cli/worker.rb +26 -0
  30. data/lib/karafka/connection/consumer.rb +29 -0
  31. data/lib/karafka/connection/listener.rb +54 -0
  32. data/lib/karafka/connection/message.rb +17 -0
  33. data/lib/karafka/connection/topic_consumer.rb +48 -0
  34. data/lib/karafka/errors.rb +50 -0
  35. data/lib/karafka/fetcher.rb +40 -0
  36. data/lib/karafka/helpers/class_matcher.rb +77 -0
  37. data/lib/karafka/helpers/multi_delegator.rb +31 -0
  38. data/lib/karafka/loader.rb +77 -0
  39. data/lib/karafka/logger.rb +52 -0
  40. data/lib/karafka/monitor.rb +82 -0
  41. data/lib/karafka/params/interchanger.rb +33 -0
  42. data/lib/karafka/params/params.rb +102 -0
  43. data/lib/karafka/patches/dry/configurable/config.rb +37 -0
  44. data/lib/karafka/process.rb +61 -0
  45. data/lib/karafka/responders/builder.rb +33 -0
  46. data/lib/karafka/responders/topic.rb +43 -0
  47. data/lib/karafka/responders/usage_validator.rb +59 -0
  48. data/lib/karafka/routing/builder.rb +89 -0
  49. data/lib/karafka/routing/route.rb +80 -0
  50. data/lib/karafka/routing/router.rb +38 -0
  51. data/lib/karafka/server.rb +53 -0
  52. data/lib/karafka/setup/config.rb +57 -0
  53. data/lib/karafka/setup/configurators/base.rb +33 -0
  54. data/lib/karafka/setup/configurators/celluloid.rb +20 -0
  55. data/lib/karafka/setup/configurators/sidekiq.rb +34 -0
  56. data/lib/karafka/setup/configurators/water_drop.rb +19 -0
  57. data/lib/karafka/setup/configurators/worker_glass.rb +13 -0
  58. data/lib/karafka/status.rb +23 -0
  59. data/lib/karafka/templates/app.rb.example +26 -0
  60. data/lib/karafka/templates/application_controller.rb.example +5 -0
  61. data/lib/karafka/templates/application_responder.rb.example +9 -0
  62. data/lib/karafka/templates/application_worker.rb.example +12 -0
  63. data/lib/karafka/templates/config.ru.example +13 -0
  64. data/lib/karafka/templates/sidekiq.yml.example +26 -0
  65. data/lib/karafka/version.rb +6 -0
  66. data/lib/karafka/workers/builder.rb +49 -0
  67. data/log/.gitkeep +0 -0
  68. metadata +267 -0
@@ -0,0 +1,26 @@
1
+ module Karafka
2
+ # Karafka framework Cli
3
+ class Cli
4
+ # Worker Karafka Cli action
5
+ class Worker < Base
6
+ desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
7
+ option aliases: 'w'
8
+
9
+ # Start the Karafka Sidekiq worker
10
+ # @param params [Array<String>] additional params that will be passed to sidekiq, that way we
11
+ # can override any default Karafka settings
12
+ def call(*params)
13
+ puts 'Starting Karafka worker'
14
+ config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
15
+ req = "-r #{Karafka.boot_file}"
16
+ env = "-e #{Karafka.env}"
17
+
18
+ cli.info
19
+
20
+ cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
21
+ puts(cmd)
22
+ exec(cmd)
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,29 @@
1
+ module Karafka
2
+ module Connection
3
+ # Class that consumes messages for which we listen
4
+ class Consumer
5
+ # Consumes a message (does something with it)
6
+ # It will execute a scheduling task from a proper controller based on a message topic
7
+ # @note This should be looped to obtain a constant listening
8
+ # @note We catch all the errors here, to make sure that none failures
9
+ # for a given consumption will affect other consumed messages
10
+ # If we would't catch it, it would propagate up until killing the Celluloid actor
11
+ # @param message [Kafka::FetchedMessage] message that was fetched by kafka
12
+ def consume(message)
13
+ controller = Karafka::Routing::Router.new(message.topic).build
14
+ # We wrap it around with our internal message format, so we don't pass around
15
+ # a raw Kafka message
16
+ controller.params = Message.new(message.topic, message.value)
17
+
18
+ Karafka.monitor.notice(self.class, controller.to_h)
19
+
20
+ controller.schedule
21
+ # This is on purpose - see the notes for this method
22
+ # rubocop:disable RescueException
23
+ rescue Exception => e
24
+ # rubocop:enable RescueException
25
+ Karafka.monitor.notice_error(self.class, e)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,54 @@
1
+ module Karafka
2
+ module Connection
3
+ # A single listener that listens to incoming messages from a single route
4
+ # @note It does not loop on itself - it needs to be executed in a loop
5
+ # @note Listener itself does nothing with the message - it will return to the block
6
+ # a raw Kafka::FetchedMessage
7
+ class Listener
8
+ include Celluloid
9
+
10
+ execute_block_on_receiver :fetch_loop
11
+
12
+ attr_reader :route
13
+
14
+ # @return [Karafka::Connection::Listener] listener instance
15
+ def initialize(route)
16
+ @route = route
17
+ end
18
+
19
+ # Opens connection, gets messages and calls a block for each of the incoming messages
20
+ # @yieldparam [Karafka::BaseController] base controller descendant
21
+ # @yieldparam [Kafka::FetchedMessage] kafka fetched message
22
+ # @note This will yield with a raw message - no preprocessing or reformatting
23
+ # @note We catch all the errors here, so they don't affect other listeners (or this one)
24
+ # so we will be able to listen and consume other incoming messages.
25
+ # Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
26
+ # won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
27
+ # Kafka connections / Internet connection issues / Etc. Business logic problems should not
28
+ # propagate this far
29
+ def fetch_loop(block)
30
+ topic_consumer.fetch_loop do |raw_message|
31
+ block.call(raw_message)
32
+ end
33
+ # This is on purpose - see the notes for this method
34
+ # rubocop:disable RescueException
35
+ rescue Exception => e
36
+ # rubocop:enable RescueException
37
+ Karafka.monitor.notice_error(self.class, e)
38
+ @topic_consumer&.stop
39
+ retry if @topic_consumer
40
+ end
41
+
42
+ private
43
+
44
+ # @return [Karafka::Connection::TopicConsumer] wrapped kafka consumer for a given topic
45
+ # consumption
46
+ # @note It adds consumer into Karafka::Server consumers pool for graceful shutdown on exit
47
+ def topic_consumer
48
+ @topic_consumer ||= TopicConsumer.new(@route).tap do |consumer|
49
+ Karafka::Server.consumers << consumer if Karafka::Server.consumers
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,17 @@
1
+ module Karafka
2
+ # Namespace that encapsulates everything related to connections
3
+ module Connection
4
+ # Single incoming Kafka message instance wrapper
5
+ class Message
6
+ attr_reader :topic, :content
7
+
8
+ # @param topic [String] topic from which this message comes
9
+ # @param content [String] raw message content (not deserialized or anything) from Kafka
10
+ # @return [Karafka::Connection::Message] incoming message instance
11
+ def initialize(topic, content)
12
+ @topic = topic
13
+ @content = content
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,48 @@
1
+ module Karafka
2
+ module Connection
3
+ # Class used as a wrapper around Ruby-Kafka to simplify additional
4
+ # features that we provide/might provide in future
5
+ class TopicConsumer
6
+ # Creates a queue consumer that will pull the data from Kafka
7
+ # @param [Karafka::Routing::Route] route details that will be used to build up a
8
+ # queue consumer instance
9
+ # @return [Karafka::Connection::QueueConsumer] queue consumer instance
10
+ def initialize(route)
11
+ @route = route
12
+ end
13
+
14
+ # Opens connection, gets messages and calls a block for each of the incoming messages
15
+ # @yieldparam [Kafka::FetchedMessage] kafka fetched message
16
+ # @note This will yield with a raw message - no preprocessing or reformatting
17
+ def fetch_loop
18
+ kafka_consumer.each_message do |message|
19
+ yield(message)
20
+ end
21
+ end
22
+
23
+ # Gracefuly stops topic consumption
24
+ def stop
25
+ kafka_consumer.stop
26
+ @kafka_consumer = nil
27
+ end
28
+
29
+ private
30
+
31
+ # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
32
+ # that is set up to consume a given routes topic
33
+ def kafka_consumer
34
+ return @kafka_consumer if @kafka_consumer
35
+
36
+ kafka = Kafka.new(
37
+ seed_brokers: ::Karafka::App.config.kafka.hosts,
38
+ logger: ::Karafka.logger,
39
+ client_id: ::Karafka::App.config.name
40
+ )
41
+
42
+ @kafka_consumer = kafka.consumer(group_id: @route.group)
43
+ @kafka_consumer.subscribe(@route.topic)
44
+ @kafka_consumer
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,50 @@
1
+ module Karafka
2
+ # Namespace used to encapsulate all the internal errors of Karafka
3
+ module Errors
4
+ # Base class for all the Karafka internal errors
5
+ class BaseError < StandardError; end
6
+
7
+ # Should be raised when we attemp to parse incoming params but parsing fails
8
+ # If this error (or its descendant) is detected, we will pass the raw message
9
+ # into params and proceed further
10
+ class ParserError < BaseError; end
11
+
12
+ # Raised when router receives topic name which does not correspond with any routes
13
+ # This should never happen because we listed only to topics defined in routes
14
+ # but theory is not always right. If you encounter this error - please contact
15
+ # Karafka maintainers
16
+ class NonMatchingRouteError < BaseError; end
17
+
18
+ # Raised when we have few controllers(inherited from Karafka::BaseController)
19
+ # with the same group name
20
+ class DuplicatedGroupError < BaseError; end
21
+
22
+ # Raised when we have few controllers(inherited from Karafka::BaseController)
23
+ # with the same topic name
24
+ class DuplicatedTopicError < BaseError; end
25
+
26
+ # Raised when we want to use topic name that has unsupported characters
27
+ class InvalidTopicName < BaseError; end
28
+
29
+ # Raised when we want to use group name that has unsupported characters
30
+ class InvalidGroupName < BaseError; end
31
+
32
+ # Raised when application does not have ApplicationWorker or other class that directly
33
+ # inherits from Karafka::BaseWorker
34
+ class BaseWorkerDescentantMissing < BaseError; end
35
+
36
+ # Raised when we want to use #respond_with in controllers but we didn't define
37
+ # (and we couldn't find) any appropriate responder for a given controller
38
+ class ResponderMissing < BaseError; end
39
+
40
+ # Raised when we want to use #respond_to in responders with a topic that we didn't register
41
+ class UnregisteredTopic < BaseError; end
42
+
43
+ # Raised when we send more than one message to a single topic but we didn't allow that when
44
+ # we were registering topic in a responder
45
+ class TopicMultipleUsage < BaseError; end
46
+
47
+ # Raised when we didn't use a topic that was defined as non-optional (required)
48
+ class UnusedResponderRequiredTopic < BaseError; end
49
+ end
50
+ end
@@ -0,0 +1,40 @@
1
+ module Karafka
2
+ # Class used to run the Karafka consumer and handle shutting down, restarting etc
3
+ # @note Creating multiple fetchers will result in having multiple connections to the same
4
+ # topics, which means that if there are no partitions, it won't use them.
5
+ class Fetcher
6
+ # Starts listening on all the listeners asynchronously
7
+ # Fetch loop should never end, which means that we won't create more actor clusters
8
+ # so we don't have to terminate them
9
+ def fetch_loop
10
+ futures = listeners.map do |listener|
11
+ listener.future.public_send(:fetch_loop, consumer)
12
+ end
13
+
14
+ futures.map(&:value)
15
+ # If anything crashes here, we need to raise the error and crush the runner because it means
16
+ # that something really bad happened
17
+ rescue => e
18
+ Karafka.monitor.notice_error(self.class, e)
19
+ Karafka::App.stop!
20
+ raise e
21
+ end
22
+
23
+ private
24
+
25
+ # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
26
+ def listeners
27
+ @listeners ||= App.routes.map do |route|
28
+ Karafka::Connection::Listener.new(route)
29
+ end
30
+ end
31
+
32
+ # @return [Proc] proc that should be processed when a message arrives
33
+ # @yieldparam message [Kafka::FetchedMessage] message from kafka (raw one)
34
+ def consumer
35
+ lambda do |message|
36
+ Karafka::Connection::Consumer.new.consume(message)
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,77 @@
1
+ module Karafka
2
+ module Helpers
3
+ # Class used to autodetect corresponding classes that are internally inside Karafka framework
4
+ # It is used among others to match:
5
+ # controller => worker
6
+ # controller => responder
7
+ class ClassMatcher
8
+ # Regexp used to remove any non classy like characters that might be in the controller
9
+ # class name (if defined dynamically, etc)
10
+ CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
11
+
12
+ # @param klass [Class] class to which we want to find a corresponding class
13
+ # @param from [String] what type of object is it (based on postfix name part)
14
+ # @param to [String] what are we looking for (based on a postfix name part)
15
+ # @example Controller that has a corresponding worker
16
+ # matcher = Karafka::Helpers::ClassMatcher.new(SuperController, 'Controller', 'Worker')
17
+ # matcher.match #=> SuperWorker
18
+ # @example Controller without a corresponding worker
19
+ # matcher = Karafka::Helpers::ClassMatcher.new(Super2Controller, 'Controller', 'Worker')
20
+ # matcher.match #=> nil
21
+ def initialize(klass, from:, to:)
22
+ @klass = klass
23
+ @from = from
24
+ @to = to
25
+ end
26
+
27
+ # @return [Class] matched class
28
+ # @return [nil] nil if we couldn't find matching class
29
+ def match
30
+ return nil if name.empty?
31
+ return nil unless scope.const_defined?(name)
32
+ matching = scope.const_get(name)
33
+ same_scope?(matching) ? matching : nil
34
+ end
35
+
36
+ # @return [String] name of a new class that we're looking for
37
+ # @note This method returns name of a class without a namespace
38
+ # @example From SuperController matching worker
39
+ # matcher.name #=> 'SuperWorker'
40
+ # @example From Namespaced::Super2Controller matching worker
41
+ # matcher.name #=> Super2Worker
42
+ def name
43
+ inflected = @klass.to_s.split('::').last.to_s
44
+ inflected.gsub!(@from, @to)
45
+ inflected.gsub!(CONSTANT_REGEXP, '')
46
+ inflected
47
+ end
48
+
49
+ # @return [Class, Module] class or module in which we're looking for a matching
50
+ def scope
51
+ scope_of(@klass)
52
+ end
53
+
54
+ private
55
+
56
+ # @param klass [Class] class for which we want to extract it's enclosing class/module
57
+ # @return [Class, Module] enclosing class/module
58
+ # @return [::Object] object if it was a root class
59
+ #
60
+ # @example Non-namespaced class
61
+ # scope_of(SuperClass) #=> Object
62
+ # @example Namespaced class
63
+ # scope_of(Abc::SuperClass) #=> Abc
64
+ def scope_of(klass)
65
+ enclosing = klass.to_s.split('::')[0...-1]
66
+ return ::Object if enclosing.empty?
67
+ ::Object.const_get(enclosing.join('::'))
68
+ end
69
+
70
+ # @param matching [Class] class of which scope we want to check
71
+ # @return [Boolean] true if the scope of class is the same as scope of matching
72
+ def same_scope?(matching)
73
+ scope == scope_of(matching)
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,31 @@
1
+ module Karafka
2
+ # Module containing classes and methods that provide some additional functionalities
3
+ module Helpers
4
+ # @note Taken from http://stackoverflow.com/questions/6407141
5
+ # Multidelegator is used to delegate calls to multiple targets
6
+ class MultiDelegator
7
+ # @param targets to which we want to delegate methods
8
+ #
9
+ def initialize(*targets)
10
+ @targets = targets
11
+ end
12
+
13
+ class << self
14
+ # @param methods names that should be delegated to
15
+ # @example Delegate write and close to STDOUT and file
16
+ # Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)
17
+ def delegate(*methods)
18
+ methods.each do |m|
19
+ define_method(m) do |*args|
20
+ @targets.map { |t| t.send(m, *args) }
21
+ end
22
+ end
23
+
24
+ self
25
+ end
26
+
27
+ alias to new
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,77 @@
1
+ module Karafka
2
+ # Loader for requiring all the files in a proper order
3
+ # Some files needs to be required before other, so it will
4
+ # try to figure it out. It will load 'base*' files first and then
5
+ # any other.
6
+ class Loader
7
+ # Order in which we want to load app files
8
+ DIRS = %w(
9
+ config/initializers
10
+ lib
11
+ app/helpers
12
+ app/inputs
13
+ app/decorators
14
+ app/models/concerns
15
+ app/models
16
+ app/responders
17
+ app/services
18
+ app/presenters
19
+ app/workers
20
+ app/controllers
21
+ app/aspects
22
+ app
23
+ ).freeze
24
+
25
+ # Will load files in a proper order (based on DIRS)
26
+ # @param [String] root path from which we want to start
27
+ def load(root)
28
+ DIRS.each do |dir|
29
+ path = File.join(root, dir)
30
+ load!(path)
31
+ end
32
+ end
33
+
34
+ # Requires all the ruby files from one path in a proper order
35
+ # @param path [String] path (dir) from which we want to load ruby files in a proper order
36
+ # @note First we load all the base files that might be used in inheritance
37
+ def load!(path)
38
+ base_load!(path)
39
+ files_load!(path)
40
+ end
41
+
42
+ # Requires all the ruby files from one relative path inside application directory
43
+ # @param relative_path [String] relative path (dir) to a file inside application directory
44
+ # from which we want to load ruby files in a proper order
45
+ def relative_load!(relative_path)
46
+ path = File.join(::Karafka.root, relative_path)
47
+ load!(path)
48
+ end
49
+
50
+ private
51
+
52
+ # Loads all the base files
53
+ # @param path [String] path (dir) from which we want to load ruby base files in a proper order
54
+ def base_load!(path)
55
+ bases = File.join(path, '**/base*.rb')
56
+ Dir[bases].sort(&method(:base_sorter)).each(&method(:require))
57
+ end
58
+
59
+ # Loads all other files (not base)
60
+ # @param path [String] path (dir) from which we want to load ruby files in a proper order
61
+ # @note Technically it will load the base files again but they are already loaded so nothing
62
+ # will happen
63
+ def files_load!(path)
64
+ files = File.join(path, '**/*.rb')
65
+ Dir[files].sort.each(&method(:require))
66
+ end
67
+
68
+ # @return [Integer] order for sorting
69
+ # @note We need sort all base files based on their position in a file tree
70
+ # so all the files that are "higher" should be loaded first
71
+ # @param str1 [String] first string for comparison
72
+ # @param str2 [String] second string for comparison
73
+ def base_sorter(str1, str2)
74
+ str1.count('/') <=> str2.count('/')
75
+ end
76
+ end
77
+ end