karafka 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +7 -0
  2. data/.console_irbrc +13 -0
  3. data/.gitignore +68 -0
  4. data/.rspec +1 -0
  5. data/.ruby-gemset +1 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +18 -0
  8. data/CHANGELOG.md +415 -0
  9. data/CODE_OF_CONDUCT.md +46 -0
  10. data/CONTRIBUTING.md +41 -0
  11. data/Gemfile +11 -0
  12. data/Gemfile.lock +123 -0
  13. data/MIT-LICENCE +18 -0
  14. data/README.md +89 -0
  15. data/bin/karafka +19 -0
  16. data/config/errors.yml +6 -0
  17. data/karafka.gemspec +37 -0
  18. data/lib/karafka.rb +78 -0
  19. data/lib/karafka/app.rb +45 -0
  20. data/lib/karafka/attributes_map.rb +67 -0
  21. data/lib/karafka/backends/inline.rb +16 -0
  22. data/lib/karafka/base_consumer.rb +68 -0
  23. data/lib/karafka/base_responder.rb +204 -0
  24. data/lib/karafka/callbacks.rb +30 -0
  25. data/lib/karafka/callbacks/config.rb +22 -0
  26. data/lib/karafka/callbacks/dsl.rb +16 -0
  27. data/lib/karafka/cli.rb +54 -0
  28. data/lib/karafka/cli/base.rb +78 -0
  29. data/lib/karafka/cli/console.rb +29 -0
  30. data/lib/karafka/cli/flow.rb +46 -0
  31. data/lib/karafka/cli/info.rb +29 -0
  32. data/lib/karafka/cli/install.rb +42 -0
  33. data/lib/karafka/cli/server.rb +66 -0
  34. data/lib/karafka/connection/client.rb +117 -0
  35. data/lib/karafka/connection/config_adapter.rb +120 -0
  36. data/lib/karafka/connection/delegator.rb +46 -0
  37. data/lib/karafka/connection/listener.rb +60 -0
  38. data/lib/karafka/consumers/callbacks.rb +54 -0
  39. data/lib/karafka/consumers/includer.rb +51 -0
  40. data/lib/karafka/consumers/responders.rb +24 -0
  41. data/lib/karafka/consumers/single_params.rb +15 -0
  42. data/lib/karafka/errors.rb +50 -0
  43. data/lib/karafka/fetcher.rb +44 -0
  44. data/lib/karafka/helpers/class_matcher.rb +78 -0
  45. data/lib/karafka/helpers/config_retriever.rb +46 -0
  46. data/lib/karafka/helpers/multi_delegator.rb +33 -0
  47. data/lib/karafka/instrumentation/listener.rb +112 -0
  48. data/lib/karafka/instrumentation/logger.rb +55 -0
  49. data/lib/karafka/instrumentation/monitor.rb +64 -0
  50. data/lib/karafka/loader.rb +28 -0
  51. data/lib/karafka/params/dsl.rb +156 -0
  52. data/lib/karafka/params/params_batch.rb +46 -0
  53. data/lib/karafka/parsers/json.rb +38 -0
  54. data/lib/karafka/patches/dry_configurable.rb +35 -0
  55. data/lib/karafka/patches/ruby_kafka.rb +34 -0
  56. data/lib/karafka/persistence/client.rb +25 -0
  57. data/lib/karafka/persistence/consumer.rb +38 -0
  58. data/lib/karafka/persistence/topic.rb +29 -0
  59. data/lib/karafka/process.rb +64 -0
  60. data/lib/karafka/responders/builder.rb +36 -0
  61. data/lib/karafka/responders/topic.rb +57 -0
  62. data/lib/karafka/routing/builder.rb +61 -0
  63. data/lib/karafka/routing/consumer_group.rb +61 -0
  64. data/lib/karafka/routing/consumer_mapper.rb +34 -0
  65. data/lib/karafka/routing/proxy.rb +37 -0
  66. data/lib/karafka/routing/router.rb +29 -0
  67. data/lib/karafka/routing/topic.rb +60 -0
  68. data/lib/karafka/routing/topic_mapper.rb +55 -0
  69. data/lib/karafka/schemas/config.rb +24 -0
  70. data/lib/karafka/schemas/consumer_group.rb +77 -0
  71. data/lib/karafka/schemas/consumer_group_topic.rb +18 -0
  72. data/lib/karafka/schemas/responder_usage.rb +39 -0
  73. data/lib/karafka/schemas/server_cli_options.rb +43 -0
  74. data/lib/karafka/server.rb +94 -0
  75. data/lib/karafka/setup/config.rb +189 -0
  76. data/lib/karafka/setup/configurators/base.rb +29 -0
  77. data/lib/karafka/setup/configurators/params.rb +25 -0
  78. data/lib/karafka/setup/configurators/water_drop.rb +32 -0
  79. data/lib/karafka/setup/dsl.rb +22 -0
  80. data/lib/karafka/status.rb +25 -0
  81. data/lib/karafka/templates/application_consumer.rb.example +6 -0
  82. data/lib/karafka/templates/application_responder.rb.example +11 -0
  83. data/lib/karafka/templates/karafka.rb.example +54 -0
  84. data/lib/karafka/version.rb +7 -0
  85. data/log/.gitkeep +0 -0
  86. metadata +301 -0
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Callbacks
5
+ # App level dsl to define callbacks
6
+ module Dsl
7
+ Callbacks::TYPES.each do |callback_type|
8
+ # Allows us to define a block, that will be executed for a given moment
9
+ # @param [Block] block that should be executed after the initialization process
10
+ define_method callback_type do |&block|
11
+ config.callbacks.send(callback_type).push block
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ # If you want to add/modify command that belongs to CLI, please review all commands
6
+ # available in cli/ directory inside Karafka source code.
7
+ #
8
+ # @note Whole Cli is built using Thor
9
+ # @see https://github.com/erikhuda/thor
10
+ class Cli < Thor
11
+ package_name 'Karafka'
12
+
13
+ class << self
14
+ # Loads all Cli commands into Thor framework
15
+ # This method should be executed before we run Karafka::Cli.start, otherwise we won't
16
+ # have any Cli commands available
17
+ def prepare
18
+ cli_commands.each do |action|
19
+ action.bind_to(self)
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ # @return [Array<Class>] Array with Cli action classes that can be used as commands
26
+ def cli_commands
27
+ constants
28
+ .map! { |object| const_get(object) }
29
+ .keep_if do |object|
30
+ object.instance_of?(Class) && (object < Cli::Base)
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ # This is kinda trick - since we don't have a autoload and other magic stuff
38
+ # like Rails does, so instead this method allows us to replace currently running
39
+ # console with a new one via Kernel.exec. It will start console with new code loaded
40
+ # Yes we know that it is not turbofast, however it is turbo convinient and small
41
+ #
42
+ # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
43
+ # so this method is only available when the Karafka console is running
44
+ #
45
+ # We skip this because this should exist and be only valid in the console
46
+ # :nocov:
47
+ if ENV['KARAFKA_CONSOLE']
48
+ # Reloads Karafka irb console session
49
+ def reload!
50
+ puts "Reloading...\n"
51
+ Kernel.exec Karafka::Cli::Console.command
52
+ end
53
+ end
54
+ # :nocov:
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ class Cli < Thor
5
+ # Base class for all the command that we want to define
6
+ # This base class provides a nicer interface to Thor and allows to easier separate single
7
+ # independent commands
8
+ # In order to define a new command you need to:
9
+ # - specify its desc
10
+ # - implement call method
11
+ #
12
+ # @example Create a dummy command
13
+ # class Dummy < Base
14
+ # self.desc = 'Dummy command'
15
+ #
16
+ # def call
17
+ # puts 'I'm doing nothing!
18
+ # end
19
+ # end
20
+ class Base
21
+ include Thor::Shell
22
+
23
+ # We can use it to call other cli methods via this object
24
+ attr_reader :cli
25
+
26
+ # @param cli [Karafka::Cli] current Karafka Cli instance
27
+ def initialize(cli)
28
+ @cli = cli
29
+ end
30
+
31
+ # This method should implement proper cli action
32
+ def call
33
+ raise NotImplementedError, 'Implement this in a subclass'
34
+ end
35
+
36
+ class << self
37
+ # Allows to set options for Thor cli
38
+ # @see https://github.com/erikhuda/thor
39
+ # @param option Single option details
40
+ def option(*option)
41
+ @options ||= []
42
+ @options << option
43
+ end
44
+
45
+ # Allows to set description of a given cli command
46
+ # @param desc [String] Description of a given cli command
47
+ def desc(desc)
48
+ @desc ||= desc
49
+ end
50
+
51
+ # This method will bind a given Cli command into Karafka Cli
52
+ # This method is a wrapper to way Thor defines its commands
53
+ # @param cli_class [Karafka::Cli] Karafka cli_class
54
+ def bind_to(cli_class)
55
+ cli_class.desc name, @desc
56
+
57
+ (@options || []).each { |option| cli_class.option(*option) }
58
+
59
+ context = self
60
+
61
+ cli_class.send :define_method, name do |*args|
62
+ context.new(self).call(*args)
63
+ end
64
+ end
65
+
66
+ private
67
+
68
+ # @return [String] downcased current class name that we use to define name for
69
+ # given Cli command
70
+ # @example for Karafka::Cli::Install
71
+ # name #=> 'install'
72
+ def name
73
+ to_s.split('::').last.downcase
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Console Karafka Cli action
7
+ class Console < Base
8
+ desc 'Start the Karafka console (short-cut alias: "c")'
9
+ option aliases: 'c'
10
+
11
+ # @return [String] Console executing command
12
+ # @example
13
+ # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
14
+ def self.command
15
+ envs = [
16
+ "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
17
+ 'KARAFKA_CONSOLE=true'
18
+ ]
19
+ "#{envs.join(' ')} bundle exec irb"
20
+ end
21
+
22
+ # Start the Karafka console
23
+ def call
24
+ cli.info
25
+ exec self.class.command
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Description of topics flow (incoming/outgoing)
7
+ class Flow < Base
8
+ desc 'Print application data flow (incoming => outgoing)'
9
+
10
+ # Print out all defined routes in alphabetical order
11
+ def call
12
+ topics.each do |topic|
13
+ any_topics = !topic.responder&.topics.nil?
14
+
15
+ if any_topics
16
+ puts "#{topic.name} =>"
17
+
18
+ topic.responder.topics.each_value do |responder_topic|
19
+ features = []
20
+ features << (responder_topic.required? ? 'always' : 'conditionally')
21
+ features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
22
+
23
+ print responder_topic.name, "(#{features.join(', ')})"
24
+ end
25
+ else
26
+ puts "#{topic.name} => (nothing)"
27
+ end
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
34
+ def topics
35
+ Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
36
+ end
37
+
38
+ # Prints a given value with label in a nice way
39
+ # @param label [String] label describing value
40
+ # @param value [String] value that should be printed
41
+ def print(label, value)
42
+ printf "%-25s %s\n", " - #{label}:", value
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Info Karafka Cli action
7
+ class Info < Base
8
+ desc 'Print configuration details and other options of your application'
9
+
10
+ # Print configuration details and other options of your application
11
+ def call
12
+ config = Karafka::App.config
13
+
14
+ info = [
15
+ "Karafka framework version: #{Karafka::VERSION}",
16
+ "Application client id: #{config.client_id}",
17
+ "Backend: #{config.backend}",
18
+ "Batch fetching: #{config.batch_fetching}",
19
+ "Batch consuming: #{config.batch_consuming}",
20
+ "Boot file: #{Karafka.boot_file}",
21
+ "Environment: #{Karafka.env}",
22
+ "Kafka seed brokers: #{config.kafka.seed_brokers}"
23
+ ]
24
+
25
+ puts(info.join("\n"))
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Install Karafka Cli action
7
+ class Install < Base
8
+ desc 'Install all required things for Karafka application in current directory'
9
+
10
+ # Directories created by default
11
+ INSTALL_DIRS = %w[
12
+ app/consumers
13
+ app/responders
14
+ config
15
+ log
16
+ tmp/pids
17
+ ].freeze
18
+
19
+ # Where should we map proper files from templates
20
+ INSTALL_FILES_MAP = {
21
+ 'karafka.rb.example' => Karafka.boot_file.basename,
22
+ 'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
23
+ 'application_responder.rb.example' => 'app/responders/application_responder.rb'
24
+ }.freeze
25
+
26
+ # Install all required things for Karafka application in current directory
27
+ def call
28
+ INSTALL_DIRS.each do |dir|
29
+ FileUtils.mkdir_p Karafka.root.join(dir)
30
+ end
31
+
32
+ INSTALL_FILES_MAP.each do |source, target|
33
+ target = Karafka.root.join(target)
34
+ next if File.exist?(target)
35
+
36
+ source = Karafka.core_root.join("templates/#{source}")
37
+ FileUtils.cp_r(source, target)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Server Karafka Cli action
7
+ class Server < Base
8
+ desc 'Start the Karafka server (short-cut alias: "s")'
9
+ option aliases: 's'
10
+ option :daemon, default: false, type: :boolean, aliases: :d
11
+ option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
12
+ option :consumer_groups, type: :array, default: nil, aliases: :g
13
+
14
+ # Start the Karafka server
15
+ def call
16
+ validate!
17
+
18
+ puts 'Starting Karafka server'
19
+ cli.info
20
+
21
+ if cli.options[:daemon]
22
+ FileUtils.mkdir_p File.dirname(cli.options[:pid])
23
+ daemonize
24
+ end
25
+
26
+ # We assign active topics on a server level, as only server is expected to listen on
27
+ # part of the topics
28
+ Karafka::Server.consumer_groups = cli.options[:consumer_groups]
29
+
30
+ # Remove pidfile on stop, just before the server instance is going to be GCed
31
+ # We want to delay the moment in which the pidfile is removed as much as we can,
32
+ # so instead of removing it after the server stops running, we rely on the gc moment
33
+ # when this object gets removed (it is a bit later), so it is closer to the actual
34
+ # system process end. We do that, so monitoring and deployment tools that rely on pids
35
+ # won't alarm or start new system process up until the current one is finished
36
+ ObjectSpace.define_finalizer(self, proc { send(:clean) })
37
+
38
+ Karafka::Server.run
39
+ end
40
+
41
+ private
42
+
43
+ # Checks the server cli configuration
44
+ # options validations in terms of app setup (topics, pid existence, etc)
45
+ def validate!
46
+ result = Schemas::ServerCliOptions.call(cli.options)
47
+ return if result.success?
48
+ raise Errors::InvalidConfiguration, result.errors
49
+ end
50
+
51
+ # Detaches current process into background and writes its pidfile
52
+ def daemonize
53
+ ::Process.daemon(true)
54
+ File.open(
55
+ cli.options[:pid],
56
+ 'w'
57
+ ) { |file| file.write(::Process.pid) }
58
+ end
59
+
60
+ # Removes a pidfile (if exist)
61
+ def clean
62
+ FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Connection
5
+ # Class used as a wrapper around Ruby-Kafka client to simplify additional
6
+ # features that we provide/might provide in future and to hide the internal implementation
7
+ class Client
8
+ extend Forwardable
9
+
10
+ def_delegator :kafka_consumer, :seek
11
+
12
+ # Creates a queue consumer client that will pull the data from Kafka
13
+ # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
14
+ # we create a client
15
+ # @return [Karafka::Connection::Client] group consumer that can subscribe to
16
+ # multiple topics
17
+ def initialize(consumer_group)
18
+ @consumer_group = consumer_group
19
+ Persistence::Client.write(self)
20
+ end
21
+
22
+ # Opens connection, gets messages and calls a block for each of the incoming messages
23
+ # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
24
+ # @note This will yield with raw messages - no preprocessing or reformatting.
25
+ def fetch_loop
26
+ settings = ConfigAdapter.consuming(consumer_group)
27
+
28
+ if consumer_group.batch_fetching
29
+ kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
30
+ else
31
+ # always yield an array of messages, so we have consistent API (always a batch)
32
+ kafka_consumer.each_message(*settings) { |message| yield([message]) }
33
+ end
34
+ rescue Kafka::ProcessingError => error
35
+ # If there was an error during consumption, we have to log it, pause current partition
36
+ # and process other things
37
+ Karafka.monitor.instrument(
38
+ 'connection.client.fetch_loop.error',
39
+ caller: self,
40
+ error: error.cause
41
+ )
42
+ pause(error.topic, error.partition)
43
+ retry
44
+ # This is on purpose - see the notes for this method
45
+ # rubocop:disable RescueException
46
+ rescue Exception => error
47
+ # rubocop:enable RescueException
48
+ Karafka.monitor.instrument(
49
+ 'connection.client.fetch_loop.error',
50
+ caller: self,
51
+ error: error
52
+ )
53
+ retry
54
+ end
55
+
56
+ # Gracefuly stops topic consumption
57
+ # @note Stopping running consumers without a really important reason is not recommended
58
+ # as until all the consumers are stopped, the server will keep running serving only
59
+ # part of the messages
60
+ def stop
61
+ @kafka_consumer&.stop
62
+ @kafka_consumer = nil
63
+ end
64
+
65
+ # Pauses fetching and consumption of a given topic partition
66
+ # @param topic [String] topic that we want to pause
67
+ # @param partition [Integer] number partition that we want to pause
68
+ def pause(topic, partition)
69
+ settings = ConfigAdapter.pausing(consumer_group)
70
+ timeout = settings[:timeout]
71
+ raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
72
+ kafka_consumer.pause(topic, partition, settings)
73
+ end
74
+
75
+ # Marks a given message as consumed and commit the offsets
76
+ # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
77
+ # that offset commit happen asap in case of a crash
78
+ # @param [Karafka::Params::Params] params message that we want to mark as processed
79
+ def mark_as_consumed(params)
80
+ kafka_consumer.mark_message_as_processed(params)
81
+ # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
82
+ # before the automatic triggers have kicked in.
83
+ kafka_consumer.commit_offsets
84
+ end
85
+
86
+ private
87
+
88
+ attr_reader :consumer_group
89
+
90
+ # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
91
+ # that is set up to consume from topics of a given consumer group
92
+ def kafka_consumer
93
+ @kafka_consumer ||= kafka.consumer(
94
+ *ConfigAdapter.consumer(consumer_group)
95
+ ).tap do |consumer|
96
+ consumer_group.topics.each do |topic|
97
+ consumer.subscribe(*ConfigAdapter.subscription(topic))
98
+ end
99
+ end
100
+ rescue Kafka::ConnectionError
101
+ # If we would not wait it would totally spam log file with failed
102
+ # attempts if Kafka is down
103
+ sleep(consumer_group.reconnect_timeout)
104
+ # We don't log and just reraise - this will be logged
105
+ # down the road
106
+ raise
107
+ end
108
+
109
+ # @return [Kafka] returns a Kafka
110
+ # @note We don't cache it internally because we cache kafka_consumer that uses kafka
111
+ # object instance
112
+ def kafka
113
+ Kafka.new(*ConfigAdapter.client(consumer_group))
114
+ end
115
+ end
116
+ end
117
+ end