karafka 1.3.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +2 -0
  3. data.tar.gz.sig +0 -0
  4. data/.coditsu/ci.yml +3 -0
  5. data/.console_irbrc +11 -0
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +69 -0
  10. data/.rspec +1 -0
  11. data/.ruby-gemset +1 -0
  12. data/.ruby-version +1 -0
  13. data/.travis.yml +36 -0
  14. data/CHANGELOG.md +540 -0
  15. data/CODE_OF_CONDUCT.md +46 -0
  16. data/CONTRIBUTING.md +41 -0
  17. data/Gemfile +12 -0
  18. data/Gemfile.lock +140 -0
  19. data/MIT-LICENCE +18 -0
  20. data/README.md +101 -0
  21. data/bin/karafka +19 -0
  22. data/certs/mensfeld.pem +25 -0
  23. data/config/errors.yml +39 -0
  24. data/karafka.gemspec +44 -0
  25. data/lib/karafka.rb +72 -0
  26. data/lib/karafka/app.rb +53 -0
  27. data/lib/karafka/attributes_map.rb +62 -0
  28. data/lib/karafka/backends/inline.rb +16 -0
  29. data/lib/karafka/base_consumer.rb +57 -0
  30. data/lib/karafka/base_responder.rb +226 -0
  31. data/lib/karafka/cli.rb +54 -0
  32. data/lib/karafka/cli/base.rb +78 -0
  33. data/lib/karafka/cli/console.rb +31 -0
  34. data/lib/karafka/cli/flow.rb +49 -0
  35. data/lib/karafka/cli/info.rb +31 -0
  36. data/lib/karafka/cli/install.rb +66 -0
  37. data/lib/karafka/cli/server.rb +71 -0
  38. data/lib/karafka/code_reloader.rb +67 -0
  39. data/lib/karafka/connection/api_adapter.rb +155 -0
  40. data/lib/karafka/connection/batch_delegator.rb +51 -0
  41. data/lib/karafka/connection/builder.rb +16 -0
  42. data/lib/karafka/connection/client.rb +117 -0
  43. data/lib/karafka/connection/listener.rb +71 -0
  44. data/lib/karafka/connection/message_delegator.rb +36 -0
  45. data/lib/karafka/consumers/callbacks.rb +71 -0
  46. data/lib/karafka/consumers/includer.rb +63 -0
  47. data/lib/karafka/consumers/metadata.rb +10 -0
  48. data/lib/karafka/consumers/responders.rb +24 -0
  49. data/lib/karafka/consumers/single_params.rb +15 -0
  50. data/lib/karafka/contracts.rb +10 -0
  51. data/lib/karafka/contracts/config.rb +21 -0
  52. data/lib/karafka/contracts/consumer_group.rb +206 -0
  53. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  54. data/lib/karafka/contracts/responder_usage.rb +54 -0
  55. data/lib/karafka/contracts/server_cli_options.rb +31 -0
  56. data/lib/karafka/errors.rb +51 -0
  57. data/lib/karafka/fetcher.rb +42 -0
  58. data/lib/karafka/helpers/class_matcher.rb +88 -0
  59. data/lib/karafka/helpers/config_retriever.rb +46 -0
  60. data/lib/karafka/helpers/inflector.rb +26 -0
  61. data/lib/karafka/helpers/multi_delegator.rb +32 -0
  62. data/lib/karafka/instrumentation/logger.rb +58 -0
  63. data/lib/karafka/instrumentation/monitor.rb +70 -0
  64. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  65. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  66. data/lib/karafka/params/builders/metadata.rb +33 -0
  67. data/lib/karafka/params/builders/params.rb +36 -0
  68. data/lib/karafka/params/builders/params_batch.rb +25 -0
  69. data/lib/karafka/params/metadata.rb +35 -0
  70. data/lib/karafka/params/params.rb +68 -0
  71. data/lib/karafka/params/params_batch.rb +61 -0
  72. data/lib/karafka/patches/ruby_kafka.rb +47 -0
  73. data/lib/karafka/persistence/client.rb +29 -0
  74. data/lib/karafka/persistence/consumers.rb +45 -0
  75. data/lib/karafka/persistence/topics.rb +48 -0
  76. data/lib/karafka/process.rb +60 -0
  77. data/lib/karafka/responders/builder.rb +36 -0
  78. data/lib/karafka/responders/topic.rb +55 -0
  79. data/lib/karafka/routing/builder.rb +89 -0
  80. data/lib/karafka/routing/consumer_group.rb +61 -0
  81. data/lib/karafka/routing/consumer_mapper.rb +34 -0
  82. data/lib/karafka/routing/proxy.rb +46 -0
  83. data/lib/karafka/routing/router.rb +29 -0
  84. data/lib/karafka/routing/topic.rb +62 -0
  85. data/lib/karafka/routing/topic_mapper.rb +53 -0
  86. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  87. data/lib/karafka/serialization/json/serializer.rb +31 -0
  88. data/lib/karafka/server.rb +86 -0
  89. data/lib/karafka/setup/config.rb +221 -0
  90. data/lib/karafka/setup/configurators/water_drop.rb +36 -0
  91. data/lib/karafka/setup/dsl.rb +21 -0
  92. data/lib/karafka/status.rb +29 -0
  93. data/lib/karafka/templates/application_consumer.rb.erb +7 -0
  94. data/lib/karafka/templates/application_responder.rb.erb +11 -0
  95. data/lib/karafka/templates/karafka.rb.erb +92 -0
  96. data/lib/karafka/version.rb +7 -0
  97. data/log/.gitkeep +0 -0
  98. metadata +336 -0
  99. metadata.gz.sig +2 -0
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ # If you want to add/modify command that belongs to CLI, please review all commands
6
+ # available in cli/ directory inside Karafka source code.
7
+ #
8
+ # @note Whole Cli is built using Thor
9
+ # @see https://github.com/erikhuda/thor
10
+ class Cli < Thor
11
+ package_name 'Karafka'
12
+
13
+ class << self
14
+ # Loads all Cli commands into Thor framework
15
+ # This method should be executed before we run Karafka::Cli.start, otherwise we won't
16
+ # have any Cli commands available
17
+ def prepare
18
+ cli_commands.each do |action|
19
+ action.bind_to(self)
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ # @return [Array<Class>] Array with Cli action classes that can be used as commands
26
+ def cli_commands
27
+ constants
28
+ .map! { |object| const_get(object) }
29
+ .keep_if do |object|
30
+ object.instance_of?(Class) && (object < Cli::Base)
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ # This is kinda trick - since we don't have a autoload and other magic stuff
38
+ # like Rails does, so instead this method allows us to replace currently running
39
+ # console with a new one via Kernel.exec. It will start console with new code loaded
40
+ # Yes, we know that it is not turbo fast, however it is turbo convenient and small
41
+ #
42
+ # Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
43
+ # so this method is only available when the Karafka console is running
44
+ #
45
+ # We skip this because this should exist and be only valid in the console
46
+ # :nocov:
47
+ if ENV['KARAFKA_CONSOLE']
48
+ # Reloads Karafka irb console session
49
+ def reload!
50
+ puts "Reloading...\n"
51
+ Kernel.exec Karafka::Cli::Console.command
52
+ end
53
+ end
54
+ # :nocov:
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ class Cli < Thor
5
+ # Base class for all the command that we want to define
6
+ # This base class provides a nicer interface to Thor and allows to easier separate single
7
+ # independent commands
8
+ # In order to define a new command you need to:
9
+ # - specify its desc
10
+ # - implement call method
11
+ #
12
+ # @example Create a dummy command
13
+ # class Dummy < Base
14
+ # self.desc = 'Dummy command'
15
+ #
16
+ # def call
17
+ # puts 'I'm doing nothing!
18
+ # end
19
+ # end
20
+ class Base
21
+ include Thor::Shell
22
+
23
+ # We can use it to call other cli methods via this object
24
+ attr_reader :cli
25
+
26
+ # @param cli [Karafka::Cli] current Karafka Cli instance
27
+ def initialize(cli)
28
+ @cli = cli
29
+ end
30
+
31
+ # This method should implement proper cli action
32
+ def call
33
+ raise NotImplementedError, 'Implement this in a subclass'
34
+ end
35
+
36
+ class << self
37
+ # Allows to set options for Thor cli
38
+ # @see https://github.com/erikhuda/thor
39
+ # @param option Single option details
40
+ def option(*option)
41
+ @options ||= []
42
+ @options << option
43
+ end
44
+
45
+ # Allows to set description of a given cli command
46
+ # @param desc [String] Description of a given cli command
47
+ def desc(desc)
48
+ @desc ||= desc
49
+ end
50
+
51
+ # This method will bind a given Cli command into Karafka Cli
52
+ # This method is a wrapper to way Thor defines its commands
53
+ # @param cli_class [Karafka::Cli] Karafka cli_class
54
+ def bind_to(cli_class)
55
+ cli_class.desc name, @desc
56
+
57
+ (@options || []).each { |option| cli_class.option(*option) }
58
+
59
+ context = self
60
+
61
+ cli_class.send :define_method, name do |*args|
62
+ context.new(self).call(*args)
63
+ end
64
+ end
65
+
66
+ private
67
+
68
+ # @return [String] downcased current class name that we use to define name for
69
+ # given Cli command
70
+ # @example for Karafka::Cli::Install
71
+ # name #=> 'install'
72
+ def name
73
+ to_s.split('::').last.downcase
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Console Karafka Cli action
7
+ class Console < Base
8
+ desc 'Start the Karafka console (short-cut alias: "c")'
9
+ option aliases: 'c'
10
+
11
+ class << self
12
+ # @return [String] Console executing command
13
+ # @example
14
+ # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
15
+ def command
16
+ envs = [
17
+ "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
18
+ 'KARAFKA_CONSOLE=true'
19
+ ]
20
+ "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
21
+ end
22
+ end
23
+
24
+ # Start the Karafka console
25
+ def call
26
+ cli.info
27
+ exec self.class.command
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Description of topics flow (incoming/outgoing)
7
+ class Flow < Base
8
+ desc 'Print application data flow (incoming => outgoing)'
9
+
10
+ # Print out all defined routes in alphabetical order
11
+ def call
12
+ topics.each do |topic|
13
+ any_topics = !topic.responder&.topics.nil?
14
+
15
+ if any_topics
16
+ puts "#{topic.name} =>"
17
+
18
+ topic.responder.topics.each_value do |responder_topic|
19
+ features = []
20
+ features << (responder_topic.required? ? 'always' : 'conditionally')
21
+
22
+ print responder_topic.name, "(#{features.join(', ')})"
23
+ end
24
+ else
25
+ puts "#{topic.name} => (nothing)"
26
+ end
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
33
+ def topics
34
+ Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
35
+ end
36
+
37
+ # Prints a given value with label in a nice way
38
+ # @param label [String] label describing value
39
+ # @param value [String] value that should be printed
40
+ def print(label, value)
41
+ printf(
42
+ "%<label>-25s %<value>s\n",
43
+ label: " - #{label}:",
44
+ value: value
45
+ )
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Info Karafka Cli action
7
+ class Info < Base
8
+ desc 'Print configuration details and other options of your application'
9
+
10
+ # Print configuration details and other options of your application
11
+ def call
12
+ config = Karafka::App.config
13
+
14
+ info = [
15
+ "Karafka version: #{Karafka::VERSION}",
16
+ "Ruby version: #{RUBY_VERSION}",
17
+ "Ruby-kafka version: #{::Kafka::VERSION}",
18
+ "Application client id: #{config.client_id}",
19
+ "Backend: #{config.backend}",
20
+ "Batch fetching: #{config.batch_fetching}",
21
+ "Batch consuming: #{config.batch_consuming}",
22
+ "Boot file: #{Karafka.boot_file}",
23
+ "Environment: #{Karafka.env}",
24
+ "Kafka seed brokers: #{config.kafka.seed_brokers}"
25
+ ]
26
+
27
+ puts(info.join("\n"))
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'erb'
4
+
5
+ module Karafka
6
+ # Karafka framework Cli
7
+ class Cli < Thor
8
+ # Install Karafka Cli action
9
+ class Install < Base
10
+ desc 'Install all required things for Karafka application in current directory'
11
+
12
+ # Directories created by default
13
+ INSTALL_DIRS = %w[
14
+ app/consumers
15
+ app/responders
16
+ app/workers
17
+ config
18
+ lib
19
+ log
20
+ tmp/pids
21
+ ].freeze
22
+
23
+ # Where should we map proper files from templates
24
+ INSTALL_FILES_MAP = {
25
+ 'karafka.rb.erb' => Karafka.boot_file.basename,
26
+ 'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
27
+ 'application_responder.rb.erb' => 'app/responders/application_responder.rb'
28
+ }.freeze
29
+
30
+ # @param args [Array] all the things that Thor CLI accepts
31
+ def initialize(*args)
32
+ super
33
+ @rails = Bundler::LockfileParser.new(
34
+ Bundler.read_file(
35
+ Bundler.default_lockfile
36
+ )
37
+ ).dependencies.key?('rails')
38
+ end
39
+
40
+ # Install all required things for Karafka application in current directory
41
+ def call
42
+ INSTALL_DIRS.each do |dir|
43
+ FileUtils.mkdir_p Karafka.root.join(dir)
44
+ end
45
+
46
+ INSTALL_FILES_MAP.each do |source, target|
47
+ target = Karafka.root.join(target)
48
+
49
+ template = File.read(Karafka.core_root.join("templates/#{source}"))
50
+ # @todo Replace with the keyword argument version once we don't have to support
51
+ # Ruby < 2.6
52
+ render = ::ERB.new(template, nil, '-').result(binding)
53
+
54
+ File.open(target, 'w') { |file| file.write(render) }
55
+ end
56
+ end
57
+
58
+ # @return [Boolean] true if we have Rails loaded
59
+ # This allows us to generate customized karafka.rb template with some tweaks specific for
60
+ # Rails
61
+ def rails?
62
+ @rails
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Karafka framework Cli
5
+ class Cli < Thor
6
+ # Server Karafka Cli action
7
+ class Server < Base
8
+ # Server config settings contract
9
+ CONTRACT = Contracts::ServerCliOptions.new.freeze
10
+
11
+ private_constant :CONTRACT
12
+
13
+ desc 'Start the Karafka server (short-cut alias: "s")'
14
+ option aliases: 's'
15
+ option :daemon, default: false, type: :boolean, aliases: :d
16
+ option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
17
+ option :consumer_groups, type: :array, default: nil, aliases: :g
18
+
19
+ # Start the Karafka server
20
+ def call
21
+ cli.info
22
+
23
+ validate!
24
+
25
+ if cli.options[:daemon]
26
+ FileUtils.mkdir_p File.dirname(cli.options[:pid])
27
+ daemonize
28
+ end
29
+
30
+ # We assign active topics on a server level, as only server is expected to listen on
31
+ # part of the topics
32
+ Karafka::Server.consumer_groups = cli.options[:consumer_groups]
33
+
34
+ # Remove pidfile on stop, just before the server instance is going to be GCed
35
+ # We want to delay the moment in which the pidfile is removed as much as we can,
36
+ # so instead of removing it after the server stops running, we rely on the gc moment
37
+ # when this object gets removed (it is a bit later), so it is closer to the actual
38
+ # system process end. We do that, so monitoring and deployment tools that rely on a pid
39
+ # won't alarm or start new system process up until the current one is finished
40
+ ObjectSpace.define_finalizer(self, proc { send(:clean) })
41
+
42
+ Karafka::Server.run
43
+ end
44
+
45
+ private
46
+
47
+ # Checks the server cli configuration
48
+ # options validations in terms of app setup (topics, pid existence, etc)
49
+ def validate!
50
+ result = CONTRACT.call(cli.options)
51
+ return if result.success?
52
+
53
+ raise Errors::InvalidConfigurationError, result.errors.to_h
54
+ end
55
+
56
+ # Detaches current process into background and writes its pidfile
57
+ def daemonize
58
+ ::Process.daemon(true)
59
+ File.open(
60
+ cli.options[:pid],
61
+ 'w'
62
+ ) { |file| file.write(::Process.pid) }
63
+ end
64
+
65
+ # Removes a pidfile (if exist)
66
+ def clean
67
+ FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Special type of a listener, that is not an instrumentation one, but one that triggers
5
+ # code reload in the development mode after each fetched batch (or message)
6
+ #
7
+ # Please refer to the development code reload sections for details on the benefits and downsides
8
+ # of the in-process code reloading
9
+ class CodeReloader
10
+ # This mutex is needed as we might have an application that has multiple consumer groups
11
+ # running in separate threads and we should not trigger reload before fully reloading the app
12
+ # in previous thread
13
+ MUTEX = Mutex.new
14
+
15
+ private_constant :MUTEX
16
+
17
+ # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
18
+ # the Rails loader, Zeitwerk or anything else that allows reloading triggering
19
+ # @param block [Proc] yields given block just before reloading. This can be used to hook custom
20
+ # reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
21
+ def initialize(*reloaders, &block)
22
+ @reloaders = reloaders
23
+ @block = block
24
+ end
25
+
26
+ # Binds to the instrumentation events and triggers reload
27
+ # @param _event [Dry::Event] empty dry event
28
+ # @note Since we de-register all the user defined objects and redraw routes, it means that
29
+ # we won't be able to do a multi-batch buffering in the development mode as each of the
30
+ # batches will be buffered on a newly created "per fetch" instance.
31
+ def on_connection_listener_fetch_loop(_event)
32
+ reload
33
+ end
34
+
35
+ private
36
+
37
+ # Triggers reload of both standard and Rails reloaders as well as expires all internals of
38
+ # Karafka, so it can be rediscovered and rebuilt
39
+ def reload
40
+ MUTEX.synchronize do
41
+ if @reloaders[0].respond_to?(:execute)
42
+ reload_with_rails
43
+ else
44
+ reload_without_rails
45
+ end
46
+ end
47
+ end
48
+
49
+ # Rails reloading procedure
50
+ def reload_with_rails
51
+ updatable = @reloaders.select(&:updated?)
52
+
53
+ return if updatable.empty?
54
+
55
+ updatable.each(&:execute)
56
+ @block&.call
57
+ Karafka::App.reload
58
+ end
59
+
60
+ # Zeitwerk and other reloaders
61
+ def reload_without_rails
62
+ @reloaders.each(&:reload)
63
+ @block&.call
64
+ Karafka::App.reload
65
+ end
66
+ end
67
+ end