karafka 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +7 -0
  2. data/.console_irbrc +13 -0
  3. data/.github/ISSUE_TEMPLATE.md +2 -0
  4. data/.gitignore +68 -0
  5. data/.rspec +1 -0
  6. data/.ruby-gemset +1 -0
  7. data/.ruby-version +1 -0
  8. data/.travis.yml +17 -0
  9. data/CHANGELOG.md +371 -0
  10. data/CODE_OF_CONDUCT.md +46 -0
  11. data/CONTRIBUTING.md +42 -0
  12. data/Gemfile +12 -0
  13. data/Gemfile.lock +111 -0
  14. data/MIT-LICENCE +18 -0
  15. data/README.md +95 -0
  16. data/bin/karafka +19 -0
  17. data/config/errors.yml +6 -0
  18. data/karafka.gemspec +35 -0
  19. data/lib/karafka.rb +68 -0
  20. data/lib/karafka/app.rb +52 -0
  21. data/lib/karafka/attributes_map.rb +67 -0
  22. data/lib/karafka/backends/inline.rb +17 -0
  23. data/lib/karafka/base_controller.rb +60 -0
  24. data/lib/karafka/base_responder.rb +185 -0
  25. data/lib/karafka/cli.rb +54 -0
  26. data/lib/karafka/cli/base.rb +78 -0
  27. data/lib/karafka/cli/console.rb +29 -0
  28. data/lib/karafka/cli/flow.rb +46 -0
  29. data/lib/karafka/cli/info.rb +29 -0
  30. data/lib/karafka/cli/install.rb +43 -0
  31. data/lib/karafka/cli/server.rb +67 -0
  32. data/lib/karafka/connection/config_adapter.rb +112 -0
  33. data/lib/karafka/connection/consumer.rb +121 -0
  34. data/lib/karafka/connection/listener.rb +51 -0
  35. data/lib/karafka/connection/processor.rb +61 -0
  36. data/lib/karafka/controllers/callbacks.rb +54 -0
  37. data/lib/karafka/controllers/includer.rb +51 -0
  38. data/lib/karafka/controllers/responders.rb +19 -0
  39. data/lib/karafka/controllers/single_params.rb +15 -0
  40. data/lib/karafka/errors.rb +43 -0
  41. data/lib/karafka/fetcher.rb +48 -0
  42. data/lib/karafka/helpers/class_matcher.rb +78 -0
  43. data/lib/karafka/helpers/config_retriever.rb +46 -0
  44. data/lib/karafka/helpers/multi_delegator.rb +33 -0
  45. data/lib/karafka/loader.rb +29 -0
  46. data/lib/karafka/logger.rb +53 -0
  47. data/lib/karafka/monitor.rb +98 -0
  48. data/lib/karafka/params/params.rb +128 -0
  49. data/lib/karafka/params/params_batch.rb +41 -0
  50. data/lib/karafka/parsers/json.rb +38 -0
  51. data/lib/karafka/patches/dry_configurable.rb +31 -0
  52. data/lib/karafka/patches/ruby_kafka.rb +34 -0
  53. data/lib/karafka/persistence/consumer.rb +25 -0
  54. data/lib/karafka/persistence/controller.rb +38 -0
  55. data/lib/karafka/process.rb +63 -0
  56. data/lib/karafka/responders/builder.rb +35 -0
  57. data/lib/karafka/responders/topic.rb +57 -0
  58. data/lib/karafka/routing/builder.rb +61 -0
  59. data/lib/karafka/routing/consumer_group.rb +61 -0
  60. data/lib/karafka/routing/consumer_mapper.rb +33 -0
  61. data/lib/karafka/routing/proxy.rb +37 -0
  62. data/lib/karafka/routing/router.rb +29 -0
  63. data/lib/karafka/routing/topic.rb +66 -0
  64. data/lib/karafka/routing/topic_mapper.rb +55 -0
  65. data/lib/karafka/schemas/config.rb +21 -0
  66. data/lib/karafka/schemas/consumer_group.rb +65 -0
  67. data/lib/karafka/schemas/consumer_group_topic.rb +18 -0
  68. data/lib/karafka/schemas/responder_usage.rb +39 -0
  69. data/lib/karafka/schemas/server_cli_options.rb +43 -0
  70. data/lib/karafka/server.rb +62 -0
  71. data/lib/karafka/setup/config.rb +163 -0
  72. data/lib/karafka/setup/configurators/base.rb +35 -0
  73. data/lib/karafka/setup/configurators/water_drop.rb +29 -0
  74. data/lib/karafka/status.rb +25 -0
  75. data/lib/karafka/templates/application_controller.rb.example +7 -0
  76. data/lib/karafka/templates/application_responder.rb.example +11 -0
  77. data/lib/karafka/templates/karafka.rb.example +41 -0
  78. data/lib/karafka/version.rb +7 -0
  79. data/log/.gitkeep +0 -0
  80. metadata +267 -0
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Patches
5
+ # Batches for Ruby Kafka gem
6
+ module RubyKafka
7
+ # This patch allows us to inject business logic in between fetches and before the consumer
8
+ # stop, so we can perform stop commit or anything else that we need since
9
+ # ruby-kafka fetch loop does not allow that directly
10
+ # We don't wan't to use poll ruby-kafka api as it brings many more problems that we would
11
+ # have to take care of. That way, nothing like that ever happens but we get the control
12
+ # over the stopping process that we need (since we're the once that initiate it for each
13
+ # thread)
14
+ def consumer_loop
15
+ super do
16
+ controllers = Karafka::Persistence::Controller
17
+ .all
18
+ .values
19
+ .flat_map(&:values)
20
+ .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
21
+
22
+ if Karafka::App.stopped?
23
+ controllers.each { |ctrl| ctrl.run_callbacks :before_stop }
24
+ Karafka::Persistence::Consumer.read.stop
25
+ else
26
+ controllers.each { |ctrl| ctrl.run_callbacks :before_poll }
27
+ yield
28
+ controllers.each { |ctrl| ctrl.run_callbacks :after_poll }
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Persistence
5
+ # Persistence layer to store current thread messages consumer for further use
6
+ class Consumer
7
+ # Thread.current key under which we store current thread messages consumer
8
+ PERSISTENCE_SCOPE = :consumer
9
+
10
+ # @param consumer [Karafka::Connection::Consumer] messages consumer of
11
+ # a current thread
12
+ # @return [Karafka::Connection::Consumer] persisted messages consumer
13
+ def self.write(consumer)
14
+ Thread.current[PERSISTENCE_SCOPE] = consumer
15
+ end
16
+
17
+ # @return [Karafka::Connection::Consumer] persisted messages consumer
18
+ # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
19
+ # but we try to use it anyway
20
+ def self.read
21
+ Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingConsumer)
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Module used to provide a persistent cache layer for Karafka components that need to be
5
+ # shared inside of a same thread
6
+ module Persistence
7
+ # Module used to provide a persistent cache across batch requests for a given
8
+ # topic and partition to store some additional details when the persistent mode
9
+ # for a given topic is turned on
10
+ class Controller
11
+ # Thread.current scope under which we store controllers data
12
+ PERSISTENCE_SCOPE = :controllers
13
+
14
+ class << self
15
+ # @return [Hash] current thread persistence scope hash with all the controllers
16
+ def all
17
+ Thread.current[PERSISTENCE_SCOPE] ||= {}
18
+ end
19
+
20
+ # Used to build (if block given) and/or fetch a current controller instance that will be
21
+ # used to process messages from a given topic and partition
22
+ # @return [Karafka::BaseController] base controller descendant
23
+ # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
24
+ # @param partition [Integer] number of partition for which we want to cache
25
+ def fetch(topic, partition)
26
+ all[topic.id] ||= {}
27
+
28
+ # We always store a current instance
29
+ if topic.persistent
30
+ all[topic.id][partition] ||= yield
31
+ else
32
+ all[topic.id][partition] = yield
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Class used to catch signals from ruby Signal class in order to manage Karafka stop
5
+ # @note There might be only one process - this class is a singleton
6
+ class Process
7
+ include Singleton
8
+
9
+ # Signal types that we handle
10
+ HANDLED_SIGNALS = %i[
11
+ SIGINT SIGQUIT SIGTERM
12
+ ].freeze
13
+
14
+ HANDLED_SIGNALS.each do |signal|
15
+ # Assigns a callback that will happen when certain signal will be send
16
+ # to Karafka server instance
17
+ # @note It does not define the callback itself -it needs to be passed in a block
18
+ # @example Define an action that should be taken on_sigint
19
+ # process.on_sigint do
20
+ # Karafka.logger.info('Log something here')
21
+ # exit
22
+ # end
23
+ define_method :"on_#{signal.to_s.downcase}" do |&block|
24
+ @callbacks[signal] << block
25
+ end
26
+ end
27
+
28
+ # Creates an instance of process and creates empty hash for callbacks
29
+ def initialize
30
+ @callbacks = {}
31
+ HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
32
+ end
33
+
34
+ # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
35
+ # @note If there are no callbacks, this method will just ignore a given signal that was sent
36
+ # @yield [Block] block of code that we want to execute and supervise
37
+ def supervise
38
+ HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
39
+ yield
40
+ end
41
+
42
+ private
43
+
44
+ # Traps a single signal and performs callbacks (if any) or just ignores this signal
45
+ # @param [Symbol] signal type that we want to catch
46
+ def trap_signal(signal)
47
+ trap(signal) do
48
+ notice_signal(signal)
49
+ (@callbacks[signal] || []).each(&:call)
50
+ end
51
+ end
52
+
53
+ # Informs monitoring about trapped signal
54
+ # @param [Symbol] signal type that we received
55
+ # @note We cannot perform logging from trap context, that's why
56
+ # we have to spin up a new thread to do this
57
+ def notice_signal(signal)
58
+ Thread.new do
59
+ Karafka.monitor.notice(self.class, signal: signal)
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Responders namespace encapsulates all the internal responder implementation parts
5
+ module Responders
6
+ # Responders builder is used to find (based on the controller class name) a responder that
7
+ # match the controller. This is used when user does not provide a responder inside routing
8
+ # but he still names responder with the same convention (and namespaces) as controller
9
+ # @example Matching responder exists
10
+ # Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
11
+ # @example Matching responder does not exist
12
+ # Karafka::Responder::Builder(NewBuildsController).build #=> nil
13
+ class Builder
14
+ # @param controller_class [Karafka::BaseController, nil] descendant of
15
+ # Karafka::BaseController
16
+ # @example Tries to find a responder that matches a given controller. If nothing found,
17
+ # will return nil (nil is accepted, because it means that a given controller don't
18
+ # pipe stuff further on)
19
+ def initialize(controller_class)
20
+ @controller_class = controller_class
21
+ end
22
+
23
+ # Tries to figure out a responder based on a controller class name
24
+ # @return [Class] Responder class (not an instance)
25
+ # @return [nil] or nil if there's no matching responding class
26
+ def build
27
+ Helpers::ClassMatcher.new(
28
+ @controller_class,
29
+ from: 'Controller',
30
+ to: 'Responder'
31
+ ).match
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Responders
5
+ # Topic describes a single topic on which we want to respond with responding requirements
6
+ # @example Define topic (required by default)
7
+ # Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
8
+ # @example Define optional topic
9
+ # Karafka::Responders::Topic.new(:topic_name, required: false)
10
+ # @example Define topic that on which we want to respond multiple times
11
+ # Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
12
+ class Topic
13
+ # Name of the topic on which we want to respond
14
+ attr_reader :name
15
+
16
+ # @param name [Symbol, String] name of a topic on which we want to respond
17
+ # @param options [Hash] non-default options for this topic
18
+ # @return [Karafka::Responders::Topic] topic description object
19
+ def initialize(name, options)
20
+ @name = name.to_s
21
+ @options = options
22
+ end
23
+
24
+ # @return [Boolean] is this a required topic (if not, it is optional)
25
+ def required?
26
+ @options.key?(:required) ? @options[:required] : true
27
+ end
28
+
29
+ # @return [Boolean] do we expect to use it multiple times in a single respond flow
30
+ def multiple_usage?
31
+ @options[:multiple_usage] || false
32
+ end
33
+
34
+ # @return [Boolean] was usage of this topic registered or not
35
+ def registered?
36
+ @options[:registered] == true
37
+ end
38
+
39
+ # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
40
+ # is safer and introduces less problems
41
+ def async?
42
+ @options.key?(:async) ? @options[:async] : false
43
+ end
44
+
45
+ # @return [Hash] hash with this topic attributes and options
46
+ def to_h
47
+ {
48
+ name: name,
49
+ multiple_usage: multiple_usage?,
50
+ required: required?,
51
+ registered: registered?,
52
+ async: async?
53
+ }
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ # Builder used as a DSL layer for building consumers and telling them which topics to consume
6
+ # @example Build a simple (most common) route
7
+ # consumers do
8
+ # topic :new_videos do
9
+ # controller NewVideosController
10
+ # end
11
+ # end
12
+ class Builder < Array
13
+ include Singleton
14
+
15
+ # Used to draw routes for Karafka
16
+ # @note After it is done drawing it will store and validate all the routes to make sure that
17
+ # they are correct and that there are no topic/group duplications (this is forbidden)
18
+ # @yield Evaluates provided block in a builder context so we can describe routes
19
+ # @example
20
+ # draw do
21
+ # topic :xyz do
22
+ # end
23
+ # end
24
+ def draw(&block)
25
+ instance_eval(&block)
26
+
27
+ each do |consumer_group|
28
+ hashed_group = consumer_group.to_h
29
+ validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
30
+ return if validation_result.success?
31
+ raise Errors::InvalidConfiguration, validation_result.errors
32
+ end
33
+ end
34
+
35
+ # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
36
+ # we want to use. Since Karafka supports multi-process setup, we need to be able
37
+ # to pick only those consumer groups that should be active in our given process context
38
+ def active
39
+ select(&:active?)
40
+ end
41
+
42
+ private
43
+
44
+ # Builds and saves given consumer group
45
+ # @param group_id [String, Symbol] name for consumer group
46
+ # @yield Evaluates a given block in a consumer group context
47
+ def consumer_group(group_id, &block)
48
+ consumer_group = ConsumerGroup.new(group_id.to_s)
49
+ self << Proxy.new(consumer_group, &block).target
50
+ end
51
+
52
+ # @param topic_name [String, Symbol] name of a topic from which we want to consumer
53
+ # @yield Evaluates a given block in a topic context
54
+ def topic(topic_name, &block)
55
+ consumer_group(topic_name) do
56
+ topic(topic_name, &block).tap(&:build)
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ # Object used to describe a single consumer group that is going to subscribe to
6
+ # given topics
7
+ # It is a part of Karafka's DSL
8
+ class ConsumerGroup
9
+ extend Helpers::ConfigRetriever
10
+
11
+ attr_reader :topics
12
+ attr_reader :id
13
+ attr_reader :name
14
+
15
+ # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
16
+ # yet have an application client_id namespace, this will be added here by default.
17
+ # We add it to make a multi-system development easier for people that don't use
18
+ # kafka and don't understand the concept of consumer groups.
19
+ def initialize(name)
20
+ @name = name
21
+ @id = Karafka::App.config.consumer_mapper.call(name)
22
+ @topics = []
23
+ end
24
+
25
+ # @return [Boolean] true if this consumer group should be active in our current process
26
+ def active?
27
+ Karafka::Server.consumer_groups.include?(name)
28
+ end
29
+
30
+ # Builds a topic representation inside of a current consumer group route
31
+ # @param name [String, Symbol] name of topic to which we want to subscribe
32
+ # @yield Evaluates a given block in a topic context
33
+ # @return [Karafka::Routing::Topic] newly built topic instance
34
+ def topic=(name, &block)
35
+ topic = Topic.new(name, self)
36
+ @topics << Proxy.new(topic, &block).target.tap(&:build)
37
+ @topics.last
38
+ end
39
+
40
+ Karafka::AttributesMap.consumer_group.each do |attribute|
41
+ config_retriever_for(attribute)
42
+ end
43
+
44
+ # Hashed version of consumer group that can be used for validation purposes
45
+ # @return [Hash] hash with consumer group attributes including serialized to hash
46
+ # topics inside of it.
47
+ def to_h
48
+ result = {
49
+ topics: topics.map(&:to_h),
50
+ id: id
51
+ }
52
+
53
+ Karafka::AttributesMap.consumer_group.each do |attribute|
54
+ result[attribute] = public_send(attribute)
55
+ end
56
+
57
+ result
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ # Default consumer mapper that builds consumer ids based on app id and consumer group name
6
+ # Different mapper can be used in case of preexisting consumer names or for applying
7
+ # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
8
+ #
9
+ # @example Mapper for using consumer groups without a client_id prefix
10
+ # module MyMapper
11
+ # def self.call(raw_consumer_group_name)
12
+ # raw_consumer_group_name
13
+ # end
14
+ # end
15
+ #
16
+ # @example Mapper for replacing "_" with "." in topic names
17
+ # module MyMapper
18
+ # def self.call(raw_consumer_group_name)
19
+ # [
20
+ # Karafka::App.config.client_id.to_s.underscope,
21
+ # raw_consumer_group_name
22
+ # ].join('_').gsub('_', '.')
23
+ # end
24
+ # end
25
+ module ConsumerMapper
26
+ # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
27
+ # @return [String] remapped final consumer group name
28
+ def self.call(raw_consumer_group_name)
29
+ "#{Karafka::App.config.client_id.to_s.underscore}_#{raw_consumer_group_name}"
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ # Proxy is used as a translation layer in between the DSL and raw topic and consumer group
6
+ # objects.
7
+ class Proxy
8
+ attr_reader :target
9
+
10
+ # We should proxy only non ? and = methods as we want to have a regular dsl
11
+ IGNORED_POSTFIXES = %w[
12
+ ?
13
+ =
14
+ !
15
+ ].freeze
16
+
17
+ # @param target [Object] target object to which we proxy any DSL call
18
+ # @yield Evaluates block in the proxy context
19
+ def initialize(target, &block)
20
+ @target = target
21
+ instance_eval(&block)
22
+ end
23
+
24
+ # Translates the no "=" DSL of routing into elements assignments on target
25
+ def method_missing(method_name, *arguments, &block)
26
+ return super unless respond_to_missing?(method_name)
27
+ @target.public_send(:"#{method_name}=", *arguments, &block)
28
+ end
29
+
30
+ # Tells whether or not a given element exists on the target
31
+ def respond_to_missing?(method_name, include_private = false)
32
+ return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
33
+ @target.respond_to?(:"#{method_name}=", include_private) || super
34
+ end
35
+ end
36
+ end
37
+ end