karafka 1.2.13 → 1.3.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +2 -0
  4. data/{.coditsu.yml → .coditsu/ci.yml} +1 -1
  5. data/.console_irbrc +1 -3
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +1 -0
  10. data/.ruby-version +1 -1
  11. data/.travis.yml +8 -21
  12. data/CHANGELOG.md +91 -18
  13. data/CONTRIBUTING.md +1 -1
  14. data/Gemfile +2 -5
  15. data/Gemfile.lock +79 -65
  16. data/README.md +10 -11
  17. data/bin/karafka +1 -1
  18. data/certs/mensfeld.pem +25 -0
  19. data/config/errors.yml +38 -5
  20. data/karafka.gemspec +14 -12
  21. data/lib/karafka.rb +8 -15
  22. data/lib/karafka/app.rb +14 -6
  23. data/lib/karafka/attributes_map.rb +5 -12
  24. data/lib/karafka/base_consumer.rb +19 -30
  25. data/lib/karafka/base_responder.rb +45 -27
  26. data/lib/karafka/cli.rb +1 -1
  27. data/lib/karafka/cli/console.rb +11 -9
  28. data/lib/karafka/cli/flow.rb +5 -2
  29. data/lib/karafka/cli/info.rb +3 -1
  30. data/lib/karafka/cli/install.rb +30 -6
  31. data/lib/karafka/cli/server.rb +11 -6
  32. data/lib/karafka/code_reloader.rb +67 -0
  33. data/lib/karafka/connection/api_adapter.rb +11 -4
  34. data/lib/karafka/connection/batch_delegator.rb +51 -0
  35. data/lib/karafka/connection/builder.rb +1 -1
  36. data/lib/karafka/connection/client.rb +30 -20
  37. data/lib/karafka/connection/listener.rb +24 -13
  38. data/lib/karafka/connection/message_delegator.rb +36 -0
  39. data/lib/karafka/consumers/callbacks.rb +32 -15
  40. data/lib/karafka/consumers/includer.rb +30 -18
  41. data/lib/karafka/consumers/metadata.rb +10 -0
  42. data/lib/karafka/consumers/responders.rb +2 -2
  43. data/lib/karafka/contracts.rb +10 -0
  44. data/lib/karafka/contracts/config.rb +21 -0
  45. data/lib/karafka/contracts/consumer_group.rb +206 -0
  46. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  47. data/lib/karafka/contracts/responder_usage.rb +54 -0
  48. data/lib/karafka/contracts/server_cli_options.rb +31 -0
  49. data/lib/karafka/errors.rb +17 -16
  50. data/lib/karafka/fetcher.rb +28 -30
  51. data/lib/karafka/helpers/class_matcher.rb +6 -2
  52. data/lib/karafka/helpers/config_retriever.rb +1 -1
  53. data/lib/karafka/helpers/inflector.rb +26 -0
  54. data/lib/karafka/helpers/multi_delegator.rb +0 -1
  55. data/lib/karafka/instrumentation/logger.rb +9 -6
  56. data/lib/karafka/instrumentation/monitor.rb +15 -9
  57. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  58. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  59. data/lib/karafka/params/builders/metadata.rb +33 -0
  60. data/lib/karafka/params/builders/params.rb +36 -0
  61. data/lib/karafka/params/builders/params_batch.rb +25 -0
  62. data/lib/karafka/params/metadata.rb +35 -0
  63. data/lib/karafka/params/params.rb +68 -0
  64. data/lib/karafka/params/params_batch.rb +35 -20
  65. data/lib/karafka/patches/ruby_kafka.rb +21 -8
  66. data/lib/karafka/persistence/client.rb +15 -11
  67. data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
  68. data/lib/karafka/persistence/topics.rb +48 -0
  69. data/lib/karafka/process.rb +0 -2
  70. data/lib/karafka/responders/topic.rb +6 -8
  71. data/lib/karafka/routing/builder.rb +36 -8
  72. data/lib/karafka/routing/consumer_group.rb +1 -1
  73. data/lib/karafka/routing/consumer_mapper.rb +9 -9
  74. data/lib/karafka/routing/proxy.rb +10 -1
  75. data/lib/karafka/routing/topic.rb +5 -3
  76. data/lib/karafka/routing/topic_mapper.rb +16 -18
  77. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  78. data/lib/karafka/serialization/json/serializer.rb +31 -0
  79. data/lib/karafka/server.rb +25 -27
  80. data/lib/karafka/setup/config.rb +63 -37
  81. data/lib/karafka/setup/configurators/water_drop.rb +7 -3
  82. data/lib/karafka/setup/dsl.rb +0 -1
  83. data/lib/karafka/status.rb +7 -3
  84. data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
  85. data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
  86. data/lib/karafka/templates/karafka.rb.erb +92 -0
  87. data/lib/karafka/version.rb +1 -1
  88. metadata +90 -57
  89. metadata.gz.sig +0 -0
  90. data/lib/karafka/callbacks.rb +0 -30
  91. data/lib/karafka/callbacks/config.rb +0 -22
  92. data/lib/karafka/callbacks/dsl.rb +0 -16
  93. data/lib/karafka/connection/delegator.rb +0 -46
  94. data/lib/karafka/instrumentation/listener.rb +0 -112
  95. data/lib/karafka/loader.rb +0 -28
  96. data/lib/karafka/params/dsl.rb +0 -158
  97. data/lib/karafka/parsers/json.rb +0 -38
  98. data/lib/karafka/patches/dry_configurable.rb +0 -33
  99. data/lib/karafka/persistence/topic.rb +0 -29
  100. data/lib/karafka/schemas/config.rb +0 -24
  101. data/lib/karafka/schemas/consumer_group.rb +0 -79
  102. data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
  103. data/lib/karafka/schemas/responder_usage.rb +0 -39
  104. data/lib/karafka/schemas/server_cli_options.rb +0 -43
  105. data/lib/karafka/setup/configurators/base.rb +0 -29
  106. data/lib/karafka/setup/configurators/params.rb +0 -25
  107. data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -1,43 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Schemas
5
- # Schema for validating correctness of the server cli command options
6
- # We validate some basics + the list of consumer_groups on which we want to use, to make
7
- # sure that all of them are defined, plus that a pidfile does not exist
8
- ServerCliOptions = Dry::Validation.Schema do
9
- configure do
10
- option :consumer_groups
11
-
12
- def self.messages
13
- super.merge(
14
- en: {
15
- errors: {
16
- consumer_groups_inclusion: 'Unknown consumer group.',
17
- pid_existence: 'Pidfile already exists.'
18
- }
19
- }
20
- )
21
- end
22
- end
23
-
24
- optional(:pid).filled(:str?)
25
- optional(:daemon).filled(:bool?)
26
- optional(:consumer_groups).filled(:array?)
27
-
28
- validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
29
- # If there were no consumer_groups declared in the server cli, it means that we will
30
- # run all of them and no need to validate them here at all
31
- if consumer_groups.nil?
32
- true
33
- else
34
- (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
35
- end
36
- end
37
-
38
- validate(pid_existence: :pid) do |pid|
39
- pid ? !File.exist?(pid) : true
40
- end
41
- end
42
- end
43
- end
@@ -1,29 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Setup
5
- # Configurators module is used to enclose all the external dependencies configurations
6
- # upon which Karafka depents
7
- class Configurators
8
- # Karafka has some components that it relies on (like Sidekiq)
9
- # We need to configure all of them only when the framework was set up.
10
- # Any class that descends from this one will be automatically invoked upon setup (after it)
11
- # @note This should be used only for internal Karafka dependencies configuration
12
- # End users configuration should go to the after_init block
13
- # @example Configure an Example class
14
- # class ExampleConfigurator < Base
15
- # def setup
16
- # ExampleClass.logger = Karafka.logger
17
- # ExampleClass.redis = config.redis
18
- # end
19
- # end
20
- class Base
21
- # @param _config [Karafka::Config] config instance
22
- # This method needs to be implemented in a subclass
23
- def self.setup(_config)
24
- raise NotImplementedError
25
- end
26
- end
27
- end
28
- end
29
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Setup
5
- class Configurators
6
- # Karafka::Params::Params are dynamically built based on user defined parent class
7
- # so we cannot just require it, we need to initialize it after user is done with
8
- # the framework configuration. This is a configurator that does exactly that.
9
- class Params < Base
10
- # Builds up Karafka::Params::Params class with user defined parent class
11
- # @param config [Karafka::Setup::Config] Config we can user to setup things
12
- def self.setup(config)
13
- return if defined? Karafka::Params::Params
14
-
15
- Karafka::Params.const_set(
16
- 'Params',
17
- Class
18
- .new(config.params_base_class)
19
- .tap { |klass| klass.include(Karafka::Params::Dsl) }
20
- )
21
- end
22
- end
23
- end
24
- end
25
- end
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Non Ruby on Rails setup
4
- ENV['RACK_ENV'] ||= 'development'
5
- ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
6
- Bundler.require(:default, ENV['KARAFKA_ENV'])
7
- Karafka::Loader.load(Karafka::App.root)
8
-
9
- # Ruby on Rails setup
10
- # Remove whole non-Rails setup that is above and uncomment the 4 lines below
11
- # ENV['RAILS_ENV'] ||= 'development'
12
- # ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
13
- # require ::File.expand_path('../config/environment', __FILE__)
14
- # Rails.application.eager_load!
15
-
16
- class KarafkaApp < Karafka::App
17
- setup do |config|
18
- config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
19
- config.client_id = 'example_app'
20
- config.backend = :inline
21
- config.batch_fetching = true
22
- # Uncomment this for Rails app integration
23
- # config.logger = Rails.logger
24
- end
25
-
26
- after_init do |config|
27
- # Put here all the things you want to do after the Karafka framework
28
- # initialization
29
- end
30
-
31
- # Comment out this part if you are not using instrumentation and/or you are not
32
- # interested in logging events for certain environments. Since instrumentation
33
- # notifications add extra boilerplate, if you want to achieve max performance,
34
- # listen to only what you really need for given environment.
35
- Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
36
-
37
- consumer_groups.draw do
38
- # topic :example do
39
- # consumer ExampleConsumer
40
- # end
41
-
42
- # consumer_group :bigger_group do
43
- # topic :test do
44
- # consumer TestConsumer
45
- # end
46
- #
47
- # topic :test2 do
48
- # consumer Test2Consumer
49
- # end
50
- # end
51
- end
52
- end
53
-
54
- KarafkaApp.boot!