karafka 1.2.13 → 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +2 -0
  4. data/{.coditsu.yml → .coditsu/ci.yml} +1 -1
  5. data/.console_irbrc +1 -3
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +1 -0
  10. data/.ruby-version +1 -1
  11. data/.travis.yml +8 -21
  12. data/CHANGELOG.md +91 -18
  13. data/CONTRIBUTING.md +1 -1
  14. data/Gemfile +2 -5
  15. data/Gemfile.lock +79 -65
  16. data/README.md +10 -11
  17. data/bin/karafka +1 -1
  18. data/certs/mensfeld.pem +25 -0
  19. data/config/errors.yml +38 -5
  20. data/karafka.gemspec +14 -12
  21. data/lib/karafka.rb +8 -15
  22. data/lib/karafka/app.rb +14 -6
  23. data/lib/karafka/attributes_map.rb +5 -12
  24. data/lib/karafka/base_consumer.rb +19 -30
  25. data/lib/karafka/base_responder.rb +45 -27
  26. data/lib/karafka/cli.rb +1 -1
  27. data/lib/karafka/cli/console.rb +11 -9
  28. data/lib/karafka/cli/flow.rb +5 -2
  29. data/lib/karafka/cli/info.rb +3 -1
  30. data/lib/karafka/cli/install.rb +30 -6
  31. data/lib/karafka/cli/server.rb +11 -6
  32. data/lib/karafka/code_reloader.rb +67 -0
  33. data/lib/karafka/connection/api_adapter.rb +11 -4
  34. data/lib/karafka/connection/batch_delegator.rb +51 -0
  35. data/lib/karafka/connection/builder.rb +1 -1
  36. data/lib/karafka/connection/client.rb +30 -20
  37. data/lib/karafka/connection/listener.rb +24 -13
  38. data/lib/karafka/connection/message_delegator.rb +36 -0
  39. data/lib/karafka/consumers/callbacks.rb +32 -15
  40. data/lib/karafka/consumers/includer.rb +30 -18
  41. data/lib/karafka/consumers/metadata.rb +10 -0
  42. data/lib/karafka/consumers/responders.rb +2 -2
  43. data/lib/karafka/contracts.rb +10 -0
  44. data/lib/karafka/contracts/config.rb +21 -0
  45. data/lib/karafka/contracts/consumer_group.rb +206 -0
  46. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  47. data/lib/karafka/contracts/responder_usage.rb +54 -0
  48. data/lib/karafka/contracts/server_cli_options.rb +31 -0
  49. data/lib/karafka/errors.rb +17 -16
  50. data/lib/karafka/fetcher.rb +28 -30
  51. data/lib/karafka/helpers/class_matcher.rb +6 -2
  52. data/lib/karafka/helpers/config_retriever.rb +1 -1
  53. data/lib/karafka/helpers/inflector.rb +26 -0
  54. data/lib/karafka/helpers/multi_delegator.rb +0 -1
  55. data/lib/karafka/instrumentation/logger.rb +9 -6
  56. data/lib/karafka/instrumentation/monitor.rb +15 -9
  57. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  58. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  59. data/lib/karafka/params/builders/metadata.rb +33 -0
  60. data/lib/karafka/params/builders/params.rb +36 -0
  61. data/lib/karafka/params/builders/params_batch.rb +25 -0
  62. data/lib/karafka/params/metadata.rb +35 -0
  63. data/lib/karafka/params/params.rb +68 -0
  64. data/lib/karafka/params/params_batch.rb +35 -20
  65. data/lib/karafka/patches/ruby_kafka.rb +21 -8
  66. data/lib/karafka/persistence/client.rb +15 -11
  67. data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
  68. data/lib/karafka/persistence/topics.rb +48 -0
  69. data/lib/karafka/process.rb +0 -2
  70. data/lib/karafka/responders/topic.rb +6 -8
  71. data/lib/karafka/routing/builder.rb +36 -8
  72. data/lib/karafka/routing/consumer_group.rb +1 -1
  73. data/lib/karafka/routing/consumer_mapper.rb +9 -9
  74. data/lib/karafka/routing/proxy.rb +10 -1
  75. data/lib/karafka/routing/topic.rb +5 -3
  76. data/lib/karafka/routing/topic_mapper.rb +16 -18
  77. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  78. data/lib/karafka/serialization/json/serializer.rb +31 -0
  79. data/lib/karafka/server.rb +25 -27
  80. data/lib/karafka/setup/config.rb +63 -37
  81. data/lib/karafka/setup/configurators/water_drop.rb +7 -3
  82. data/lib/karafka/setup/dsl.rb +0 -1
  83. data/lib/karafka/status.rb +7 -3
  84. data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
  85. data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
  86. data/lib/karafka/templates/karafka.rb.erb +92 -0
  87. data/lib/karafka/version.rb +1 -1
  88. metadata +90 -57
  89. metadata.gz.sig +0 -0
  90. data/lib/karafka/callbacks.rb +0 -30
  91. data/lib/karafka/callbacks/config.rb +0 -22
  92. data/lib/karafka/callbacks/dsl.rb +0 -16
  93. data/lib/karafka/connection/delegator.rb +0 -46
  94. data/lib/karafka/instrumentation/listener.rb +0 -112
  95. data/lib/karafka/loader.rb +0 -28
  96. data/lib/karafka/params/dsl.rb +0 -158
  97. data/lib/karafka/parsers/json.rb +0 -38
  98. data/lib/karafka/patches/dry_configurable.rb +0 -33
  99. data/lib/karafka/persistence/topic.rb +0 -29
  100. data/lib/karafka/schemas/config.rb +0 -24
  101. data/lib/karafka/schemas/consumer_group.rb +0 -79
  102. data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
  103. data/lib/karafka/schemas/responder_usage.rb +0 -39
  104. data/lib/karafka/schemas/server_cli_options.rb +0 -43
  105. data/lib/karafka/setup/configurators/base.rb +0 -29
  106. data/lib/karafka/setup/configurators/params.rb +0 -25
  107. data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -1,43 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Schemas
5
- # Schema for validating correctness of the server cli command options
6
- # We validate some basics + the list of consumer_groups on which we want to use, to make
7
- # sure that all of them are defined, plus that a pidfile does not exist
8
- ServerCliOptions = Dry::Validation.Schema do
9
- configure do
10
- option :consumer_groups
11
-
12
- def self.messages
13
- super.merge(
14
- en: {
15
- errors: {
16
- consumer_groups_inclusion: 'Unknown consumer group.',
17
- pid_existence: 'Pidfile already exists.'
18
- }
19
- }
20
- )
21
- end
22
- end
23
-
24
- optional(:pid).filled(:str?)
25
- optional(:daemon).filled(:bool?)
26
- optional(:consumer_groups).filled(:array?)
27
-
28
- validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
29
- # If there were no consumer_groups declared in the server cli, it means that we will
30
- # run all of them and no need to validate them here at all
31
- if consumer_groups.nil?
32
- true
33
- else
34
- (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
35
- end
36
- end
37
-
38
- validate(pid_existence: :pid) do |pid|
39
- pid ? !File.exist?(pid) : true
40
- end
41
- end
42
- end
43
- end
@@ -1,29 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Setup
5
- # Configurators module is used to enclose all the external dependencies configurations
6
- # upon which Karafka depents
7
- class Configurators
8
- # Karafka has some components that it relies on (like Sidekiq)
9
- # We need to configure all of them only when the framework was set up.
10
- # Any class that descends from this one will be automatically invoked upon setup (after it)
11
- # @note This should be used only for internal Karafka dependencies configuration
12
- # End users configuration should go to the after_init block
13
- # @example Configure an Example class
14
- # class ExampleConfigurator < Base
15
- # def setup
16
- # ExampleClass.logger = Karafka.logger
17
- # ExampleClass.redis = config.redis
18
- # end
19
- # end
20
- class Base
21
- # @param _config [Karafka::Config] config instance
22
- # This method needs to be implemented in a subclass
23
- def self.setup(_config)
24
- raise NotImplementedError
25
- end
26
- end
27
- end
28
- end
29
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Setup
5
- class Configurators
6
- # Karafka::Params::Params are dynamically built based on user defined parent class
7
- # so we cannot just require it, we need to initialize it after user is done with
8
- # the framework configuration. This is a configurator that does exactly that.
9
- class Params < Base
10
- # Builds up Karafka::Params::Params class with user defined parent class
11
- # @param config [Karafka::Setup::Config] Config we can user to setup things
12
- def self.setup(config)
13
- return if defined? Karafka::Params::Params
14
-
15
- Karafka::Params.const_set(
16
- 'Params',
17
- Class
18
- .new(config.params_base_class)
19
- .tap { |klass| klass.include(Karafka::Params::Dsl) }
20
- )
21
- end
22
- end
23
- end
24
- end
25
- end
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Non Ruby on Rails setup
4
- ENV['RACK_ENV'] ||= 'development'
5
- ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
6
- Bundler.require(:default, ENV['KARAFKA_ENV'])
7
- Karafka::Loader.load(Karafka::App.root)
8
-
9
- # Ruby on Rails setup
10
- # Remove whole non-Rails setup that is above and uncomment the 4 lines below
11
- # ENV['RAILS_ENV'] ||= 'development'
12
- # ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
13
- # require ::File.expand_path('../config/environment', __FILE__)
14
- # Rails.application.eager_load!
15
-
16
- class KarafkaApp < Karafka::App
17
- setup do |config|
18
- config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
19
- config.client_id = 'example_app'
20
- config.backend = :inline
21
- config.batch_fetching = true
22
- # Uncomment this for Rails app integration
23
- # config.logger = Rails.logger
24
- end
25
-
26
- after_init do |config|
27
- # Put here all the things you want to do after the Karafka framework
28
- # initialization
29
- end
30
-
31
- # Comment out this part if you are not using instrumentation and/or you are not
32
- # interested in logging events for certain environments. Since instrumentation
33
- # notifications add extra boilerplate, if you want to achieve max performance,
34
- # listen to only what you really need for given environment.
35
- Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
36
-
37
- consumer_groups.draw do
38
- # topic :example do
39
- # consumer ExampleConsumer
40
- # end
41
-
42
- # consumer_group :bigger_group do
43
- # topic :test do
44
- # consumer TestConsumer
45
- # end
46
- #
47
- # topic :test2 do
48
- # consumer Test2Consumer
49
- # end
50
- # end
51
- end
52
- end
53
-
54
- KarafkaApp.boot!