karafka 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +68 -0
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +6 -0
  6. data/CHANGELOG.md +202 -0
  7. data/Gemfile +8 -0
  8. data/Gemfile.lock +216 -0
  9. data/MIT-LICENCE +18 -0
  10. data/README.md +831 -0
  11. data/Rakefile +17 -0
  12. data/bin/karafka +7 -0
  13. data/karafka.gemspec +34 -0
  14. data/lib/karafka.rb +73 -0
  15. data/lib/karafka/app.rb +45 -0
  16. data/lib/karafka/base_controller.rb +162 -0
  17. data/lib/karafka/base_responder.rb +118 -0
  18. data/lib/karafka/base_worker.rb +41 -0
  19. data/lib/karafka/capistrano.rb +2 -0
  20. data/lib/karafka/capistrano/karafka.cap +84 -0
  21. data/lib/karafka/cli.rb +52 -0
  22. data/lib/karafka/cli/base.rb +74 -0
  23. data/lib/karafka/cli/console.rb +23 -0
  24. data/lib/karafka/cli/flow.rb +46 -0
  25. data/lib/karafka/cli/info.rb +26 -0
  26. data/lib/karafka/cli/install.rb +45 -0
  27. data/lib/karafka/cli/routes.rb +39 -0
  28. data/lib/karafka/cli/server.rb +59 -0
  29. data/lib/karafka/cli/worker.rb +26 -0
  30. data/lib/karafka/connection/consumer.rb +29 -0
  31. data/lib/karafka/connection/listener.rb +54 -0
  32. data/lib/karafka/connection/message.rb +17 -0
  33. data/lib/karafka/connection/topic_consumer.rb +48 -0
  34. data/lib/karafka/errors.rb +50 -0
  35. data/lib/karafka/fetcher.rb +40 -0
  36. data/lib/karafka/helpers/class_matcher.rb +77 -0
  37. data/lib/karafka/helpers/multi_delegator.rb +31 -0
  38. data/lib/karafka/loader.rb +77 -0
  39. data/lib/karafka/logger.rb +52 -0
  40. data/lib/karafka/monitor.rb +82 -0
  41. data/lib/karafka/params/interchanger.rb +33 -0
  42. data/lib/karafka/params/params.rb +102 -0
  43. data/lib/karafka/patches/dry/configurable/config.rb +37 -0
  44. data/lib/karafka/process.rb +61 -0
  45. data/lib/karafka/responders/builder.rb +33 -0
  46. data/lib/karafka/responders/topic.rb +43 -0
  47. data/lib/karafka/responders/usage_validator.rb +59 -0
  48. data/lib/karafka/routing/builder.rb +89 -0
  49. data/lib/karafka/routing/route.rb +80 -0
  50. data/lib/karafka/routing/router.rb +38 -0
  51. data/lib/karafka/server.rb +53 -0
  52. data/lib/karafka/setup/config.rb +57 -0
  53. data/lib/karafka/setup/configurators/base.rb +33 -0
  54. data/lib/karafka/setup/configurators/celluloid.rb +20 -0
  55. data/lib/karafka/setup/configurators/sidekiq.rb +34 -0
  56. data/lib/karafka/setup/configurators/water_drop.rb +19 -0
  57. data/lib/karafka/setup/configurators/worker_glass.rb +13 -0
  58. data/lib/karafka/status.rb +23 -0
  59. data/lib/karafka/templates/app.rb.example +26 -0
  60. data/lib/karafka/templates/application_controller.rb.example +5 -0
  61. data/lib/karafka/templates/application_responder.rb.example +9 -0
  62. data/lib/karafka/templates/application_worker.rb.example +12 -0
  63. data/lib/karafka/templates/config.ru.example +13 -0
  64. data/lib/karafka/templates/sidekiq.yml.example +26 -0
  65. data/lib/karafka/version.rb +6 -0
  66. data/lib/karafka/workers/builder.rb +49 -0
  67. data/log/.gitkeep +0 -0
  68. metadata +267 -0
@@ -0,0 +1,52 @@
1
+ module Karafka
2
+ # Default logger for Event Delegator
3
+ # @note It uses ::Logger features - providing basic logging
4
+ class Logger < ::Logger
5
+ # Map containing informations about log level for given environment
6
+ ENV_MAP = {
7
+ 'production' => ::Logger::ERROR,
8
+ 'test' => ::Logger::ERROR,
9
+ 'development' => ::Logger::INFO,
10
+ 'debug' => ::Logger::DEBUG,
11
+ default: ::Logger::INFO
12
+ }.freeze
13
+
14
+ class << self
15
+ # Returns a logger instance with appropriate settings, log level and environment
16
+ def instance
17
+ ensure_dir_exists
18
+ instance = new(target)
19
+ instance.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
20
+ instance
21
+ end
22
+
23
+ private
24
+
25
+ # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
26
+ # to which we will be writtng logs
27
+ # We use this approach to log stuff to file and to the STDOUT at the same time
28
+ def target
29
+ Karafka::Helpers::MultiDelegator
30
+ .delegate(:write, :close)
31
+ .to(STDOUT, file)
32
+ end
33
+
34
+ # Makes sure the log directory exists
35
+ def ensure_dir_exists
36
+ dir = File.dirname(log_path)
37
+ FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
38
+ end
39
+
40
+ # @return [Pathname] Path to a file to which we should log
41
+ def log_path
42
+ Karafka::App.root.join("log/#{Karafka.env}.log")
43
+ end
44
+
45
+ # @return [File] file to which we want to write our logs
46
+ # @note File is being opened in append mode ('a')
47
+ def file
48
+ File.open(log_path, 'a')
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,82 @@
1
+ module Karafka
2
+ # Monitor is used to hookup external monitoring services to monitor how Karafka works
3
+ # It provides a standarized API for checking incoming messages/enqueueing etc
4
+ # By default it implements logging functionalities but can be replaced with any more
5
+ # sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
6
+ # @note This class acts as a singleton because we are only permitted to have single monitor
7
+ # per running process (just as logger)
8
+ # Keep in mind, that if you create your own monitor object, you will have to implement also
9
+ # logging functionality (or just inherit, super and do whatever you want)
10
+ class Monitor
11
+ include Singleton
12
+
13
+ # This method is executed in many important places in the code (during data flow), like
14
+ # the moment before #perform_async, etc. For full list just grep for 'monitor.notice'
15
+ # @param caller_class [Class] class of object that executed this call
16
+ # @param options [Hash] hash with options that we passed to notice. It differs based
17
+ # on of who and when is calling
18
+ # @note We don't provide a name of method in which this was called, because we can take
19
+ # it directly from Ruby (see #caller_label method of this class for more details)
20
+ # @example Notice about consuming with controller_class
21
+ # Karafka.monitor.notice(self.class, controller_class: controller_class)
22
+ # @example Notice about terminating with a signal
23
+ # Karafka.monitor.notice(self.class, signal: signal)
24
+ def notice(caller_class, options = {})
25
+ logger.info("#{caller_class}##{caller_label} with #{options}")
26
+ end
27
+
28
+ # This method is executed when we want to notify about an error that happened somewhere
29
+ # in the system
30
+ # @param caller_class [Class] class of object that executed this call
31
+ # @param e [Exception] exception that was raised
32
+ # @note We don't provide a name of method in which this was called, because we can take
33
+ # it directly from Ruby (see #caller_label method of this class for more details)
34
+ # @example Notify about error
35
+ # Karafka.monitor.notice(self.class, e)
36
+ def notice_error(caller_class, e)
37
+ caller_exceptions_map.each do |level, types|
38
+ next unless types.include?(caller_class)
39
+
40
+ return logger.public_send(level, e)
41
+ end
42
+
43
+ logger.info(e)
44
+ end
45
+
46
+ private
47
+
48
+ # @return [Hash] Hash containing informations on which level of notification should
49
+ # we use for exceptions that happen in certain parts of Karafka
50
+ # @note Keep in mind that any not handled here class should be logged with info
51
+ # @note Those are not maps of exceptions classes but of classes that were callers of this
52
+ # particular exception
53
+ def caller_exceptions_map
54
+ @caller_exceptions_map ||= {
55
+ error: [
56
+ Karafka::Connection::Consumer,
57
+ Karafka::Connection::Listener,
58
+ Karafka::Params::Params
59
+ ],
60
+ fatal: [
61
+ Karafka::Fetcher
62
+ ]
63
+ }
64
+ end
65
+
66
+ # @return [String] label of method that invoked #notice or #notice_error
67
+ # @example Check label of method that invoked #notice
68
+ # caller_label #=> 'fetch'
69
+ # @example Check label of method that invoked #notice in a block
70
+ # caller_label #=> 'block in fetch'
71
+ # @example Check label of method that invoked #notice_error
72
+ # caller_label #=> 'rescue in target'
73
+ def caller_label
74
+ caller_locations(1, 2)[1].label
75
+ end
76
+
77
+ # @return [Logger] logger instance
78
+ def logger
79
+ Karafka.logger
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,33 @@
1
+ module Karafka
2
+ module Params
3
+ # Interchangers allow us to format/encode/pack data that is being send to perform_async
4
+ # This is meant to target mostly issues with data encoding like this one:
5
+ # https://github.com/mperham/sidekiq/issues/197
6
+ # Each custom interchanger should implement following methods:
7
+ # - load - it is meant to encode params before they get stored inside Redis
8
+ # - parse - decoded params back to a hash format that we can use
9
+ class Interchanger
10
+ class << self
11
+ # @param params [Karafka::Params::Params] Karafka params object
12
+ # @note Params might not be parsed because of lazy loading feature. If you implement your
13
+ # own interchanger logic, this method needs to return data that can be converted to
14
+ # json with default Sidekiqs logic
15
+ # @return [Karafka::Params::Params] same as input. We assume that our incoming data is
16
+ # jsonable-safe and we can rely on a direct Sidekiq encoding logic
17
+ def load(params)
18
+ params
19
+ end
20
+
21
+ # @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
22
+ # @note Hash is what we need to build Karafka::Params::Params, so we do nothing
23
+ # with it. If you implement your own interchanger logic, this method needs to return
24
+ # a hash with appropriate data that will be used to build Karafka::Params::Params
25
+ # @return [Hash] We return exactly what we received. We rely on sidekiqs default
26
+ # interchanging format
27
+ def parse(params)
28
+ params
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,102 @@
1
+ module Karafka
2
+ # Params namespace encapsulating all the logic that is directly related to params handling
3
+ module Params
4
+ # Class-wrapper for hash with indifferent access with additional lazy loading feature
5
+ # It provides lazy loading not only until the first usage, but also allows us to skip
6
+ # using parser until we execute our logic inside worker. That way we can operate with
7
+ # heavy-parsing data without slowing down the whole application. If we won't use
8
+ # params in before_enqueue (or if we don't us before_enqueue at all), it will make
9
+ # Karafka faster, because it will pass data as it is directly to Sidekiq
10
+ class Params < HashWithIndifferentAccess
11
+ class << self
12
+ # We allow building instances only via the #build method
13
+ private_class_method :new
14
+
15
+ # @param message [Karafka::Connection::Message, Hash] message that we get out of Kafka
16
+ # in case of building params inside main Karafka process in
17
+ # Karafka::Connection::Consumer, or a hash when we retrieve data from Sidekiq
18
+ # @param controller [Karafka::BaseController] Karafka's base controllers descendant
19
+ # instance that wants to use params
20
+ # @return [Karafka::Params::Params] Karafka params object not yet used parser for
21
+ # retrieving data that we've got from Kafka
22
+ # @example Build params instance from a hash
23
+ # Karafka::Params::Params.build({ key: 'value' }, DataController.new) #=> params object
24
+ # @example Build params instance from a Karafka::Connection::Message object
25
+ # Karafka::Params::Params.build(message, IncomingController.new) #=> params object
26
+ def build(message, controller)
27
+ # Hash case happens inside workers
28
+ if message.is_a?(Hash)
29
+ defaults(controller).merge!(message)
30
+ else
31
+ # This happens inside Karafka::Connection::Consumer
32
+ defaults(controller).merge!(
33
+ parsed: false,
34
+ received_at: Time.now,
35
+ content: message.content
36
+ )
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ # @param controller [Karafka::BaseController] Karafka's base controllers descendant
43
+ # instance that wants to use params
44
+ # @return [Karafka::Params::Params] freshly initialized only with default values object
45
+ # that can be populated with incoming data
46
+ def defaults(controller)
47
+ # We initialize some default values that will be used both in Karafka main process and
48
+ # inside workers
49
+ new(
50
+ controller: controller.class,
51
+ worker: controller.worker,
52
+ parser: controller.parser,
53
+ topic: controller.topic
54
+ )
55
+ end
56
+ end
57
+
58
+ # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
59
+ # retrieve data, parser will be executed to parse data. Output of parsing will be merged
60
+ # to the current object. This object will be also marked as already parsed, so we won't
61
+ # parse it again.
62
+ def retrieve
63
+ return self if self[:parsed]
64
+
65
+ merge!(parse(delete(:content)))
66
+ end
67
+
68
+ # Overwritten merge! method - it behaves differently for keys that are the same in our hash
69
+ # and in a other_hash - it will not replace keys that are the same in our hash
70
+ # and in the other one
71
+ # @param other_hash [Hash, HashWithIndifferentAccess] hash that we want to merge into current
72
+ # @return [Karafka::Params::Params] our parameters hash with merged values
73
+ # @example Merge with hash without same keys
74
+ # new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
75
+ # @example Merge with hash with same keys (symbol based)
76
+ # new(a: 1).merge!(a: 2) #=> { a: 1 }
77
+ # @example Merge with hash with same keys (string based)
78
+ # new(a: 1).merge!('a' => 2) #=> { a: 1 }
79
+ # @example Merge with hash with same keys (current string based)
80
+ # new('a' => 1).merge!(a: 2) #=> { a: 1 }
81
+ def merge!(other_hash)
82
+ super(other_hash) { |_key, base_value, _new_value| base_value }
83
+ end
84
+
85
+ private
86
+
87
+ # @param content [String] Raw data that we want to parse using controller's parser
88
+ # @note If something goes wrong, it will return raw data in a hash with a message key
89
+ # @return [Hash] parsed data or a hash with message key containing raw data if something
90
+ # went wrong during parsing
91
+ def parse(content)
92
+ self[:parser].parse(content)
93
+ # We catch both of them, because for default JSON - we use JSON parser directly
94
+ rescue ::Karafka::Errors::ParserError, JSON::ParserError => e
95
+ Karafka.monitor.notice_error(self.class, e)
96
+ return { message: content }
97
+ ensure
98
+ self[:parsed] = true
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,37 @@
1
+ # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
2
+ # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
3
+ module Dry
4
+ # Configurable module for Dry-Configurable
5
+ module Configurable
6
+ # Config node instance struct
7
+ class Config
8
+ # @param args [Array] All arguments that a Struct accepts
9
+ def initialize(*args)
10
+ super
11
+ setup_dynamics
12
+ end
13
+
14
+ private
15
+
16
+ # Method that sets up all the proc based lazy evaluated dynamic config values
17
+ def setup_dynamics
18
+ each_pair do |key, value|
19
+ next unless value.is_a?(Proc)
20
+
21
+ rebuild(key)
22
+ end
23
+ end
24
+
25
+ # Method that rebuilds a given accessor, so when it consists a proc value, it will
26
+ # evaluate it upon return
27
+ # @param method_name [Symbol] name of an accessor that we want to rebuild
28
+ def rebuild(method_name)
29
+ metaclass = class << self; self; end
30
+
31
+ metaclass.send(:define_method, method_name) do
32
+ super().is_a?(Proc) ? super().call : super()
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,61 @@
1
+ module Karafka
2
+ # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
3
+ # @note There might be only one process - this class is a singleton
4
+ class Process
5
+ include Singleton
6
+
7
+ # Signal types that we handle
8
+ HANDLED_SIGNALS = %i(
9
+ SIGINT SIGQUIT
10
+ ).freeze
11
+
12
+ HANDLED_SIGNALS.each do |signal|
13
+ # Assigns a callback that will happen when certain signal will be send
14
+ # to Karafka server instance
15
+ # @note It does not define the callback itself -it needs to be passed in a block
16
+ # @example Define an action that should be taken on_sigint
17
+ # process.on_sigint do
18
+ # Karafka.logger.info('Log something here')
19
+ # exit
20
+ # end
21
+ define_method :"on_#{signal.to_s.downcase}" do |&block|
22
+ @callbacks[signal] << block
23
+ end
24
+ end
25
+
26
+ # Creates an instance of process and creates empty hash for callbacks
27
+ def initialize
28
+ @callbacks = {}
29
+ HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
30
+ end
31
+
32
+ # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
33
+ # @note If there are no callbacks, this method will just ignore a given signal that was sent
34
+ # @yield [Block] block of code that we want to execute and supervise
35
+ def supervise
36
+ HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
37
+ yield
38
+ end
39
+
40
+ private
41
+
42
+ # Traps a single signal and performs callbacks (if any) or just ignores this signal
43
+ # @param [Symbol] signal type that we want to catch
44
+ def trap_signal(signal)
45
+ trap(signal) do
46
+ notice_signal(signal)
47
+ (@callbacks[signal] || []).each(&:call)
48
+ end
49
+ end
50
+
51
+ # Informs monitoring about trapped signal
52
+ # @param [Symbol] signal type that we received
53
+ # @note We cannot perform logging from trap context, that's why
54
+ # we have to spin up a new thread to do this
55
+ def notice_signal(signal)
56
+ Thread.new do
57
+ Karafka.monitor.notice(self.class, signal: signal)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,33 @@
1
+ module Karafka
2
+ # Responders namespace encapsulates all the internal responder implementation parts
3
+ module Responders
4
+ # Responders builder is used to find (based on the controller class name) a responder that
5
+ # match the controller. This is used when user does not provide a responder inside routing
6
+ # but he still names responder with the same convention (and namespaces) as controller
7
+ # @example Matching responder exists
8
+ # Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
9
+ # @example Matching responder does not exist
10
+ # Karafka::Responder::Builder(NewBuildsController).build #=> nil
11
+ class Builder
12
+ # @param controller_class [Karafka::BaseController, nil] descendant of
13
+ # Karafka::BaseController
14
+ # @example Tries to find a responder that matches a given controller. If nothing found,
15
+ # will return nil (nil is accepted, because it means that a given controller don't
16
+ # pipe stuff further on)
17
+ def initialize(controller_class)
18
+ @controller_class = controller_class
19
+ end
20
+
21
+ # Tries to figure out a responder based on a controller class name
22
+ # @return [Class] Responder class (not an instance)
23
+ # @return [nil] or nil if there's no matching responding class
24
+ def build
25
+ Helpers::ClassMatcher.new(
26
+ @controller_class,
27
+ from: 'Controller',
28
+ to: 'Responder'
29
+ ).match
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,43 @@
1
+ module Karafka
2
+ module Responders
3
+ # Topic describes a single topic on which we want to respond with responding requirements
4
+ # @example Define topic (required by default)
5
+ # Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
6
+ # @example Define optional topic
7
+ # Karafka::Responders::Topic.new(:topic_name, optional: true)
8
+ # @example Define topic that on which we want to respond multiple times
9
+ # Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
10
+ class Topic
11
+ # Name of the topic on which we want to respond
12
+ attr_reader :name
13
+
14
+ # @param name [Symbol, String] name of a topic on which we want to respond
15
+ # @param options [Hash] non-default options for this topic
16
+ # @return [Karafka::Responders::Topic] topic description object
17
+ def initialize(name, options)
18
+ @name = name.to_s
19
+ @options = options
20
+ validate!
21
+ end
22
+
23
+ # @return [Boolean] is this a required topic (if not, it is optional)
24
+ def required?
25
+ return false if @options[:optional]
26
+ @options[:required] || true
27
+ end
28
+
29
+ # @return [Boolean] do we expect to use it multiple times in a single respond flow
30
+ def multiple_usage?
31
+ @options[:multiple_usage] || false
32
+ end
33
+
34
+ private
35
+
36
+ # Checks topic name with the same regexp as routing
37
+ # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
38
+ def validate!
39
+ raise Errors::InvalidTopicName, name if Routing::Route::NAME_FORMAT !~ name
40
+ end
41
+ end
42
+ end
43
+ end