karafka 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +5 -5
  2. data/.ruby-version +1 -1
  3. data/.travis.yml +1 -0
  4. data/CHANGELOG.md +46 -2
  5. data/CONTRIBUTING.md +5 -6
  6. data/Gemfile +1 -2
  7. data/Gemfile.lock +41 -29
  8. data/README.md +13 -19
  9. data/karafka.gemspec +6 -4
  10. data/lib/karafka.rb +17 -7
  11. data/lib/karafka/app.rb +8 -15
  12. data/lib/karafka/attributes_map.rb +1 -1
  13. data/lib/karafka/backends/inline.rb +1 -2
  14. data/lib/karafka/{base_controller.rb → base_consumer.rb} +19 -11
  15. data/lib/karafka/base_responder.rb +34 -15
  16. data/lib/karafka/callbacks.rb +30 -0
  17. data/lib/karafka/callbacks/config.rb +22 -0
  18. data/lib/karafka/callbacks/dsl.rb +16 -0
  19. data/lib/karafka/cli/install.rb +2 -3
  20. data/lib/karafka/cli/server.rb +0 -1
  21. data/lib/karafka/connection/{consumer.rb → client.rb} +32 -36
  22. data/lib/karafka/connection/config_adapter.rb +14 -6
  23. data/lib/karafka/connection/delegator.rb +46 -0
  24. data/lib/karafka/connection/listener.rb +22 -13
  25. data/lib/karafka/{controllers → consumers}/callbacks.rb +9 -9
  26. data/lib/karafka/consumers/includer.rb +51 -0
  27. data/lib/karafka/consumers/responders.rb +24 -0
  28. data/lib/karafka/{controllers → consumers}/single_params.rb +3 -3
  29. data/lib/karafka/errors.rb +10 -3
  30. data/lib/karafka/fetcher.rb +30 -34
  31. data/lib/karafka/helpers/class_matcher.rb +8 -8
  32. data/lib/karafka/helpers/config_retriever.rb +2 -2
  33. data/lib/karafka/instrumentation/listener.rb +112 -0
  34. data/lib/karafka/instrumentation/logger.rb +55 -0
  35. data/lib/karafka/instrumentation/monitor.rb +64 -0
  36. data/lib/karafka/loader.rb +0 -1
  37. data/lib/karafka/params/{params.rb → dsl.rb} +71 -43
  38. data/lib/karafka/params/params_batch.rb +7 -2
  39. data/lib/karafka/patches/dry_configurable.rb +6 -2
  40. data/lib/karafka/patches/ruby_kafka.rb +10 -10
  41. data/lib/karafka/persistence/client.rb +25 -0
  42. data/lib/karafka/persistence/consumer.rb +27 -14
  43. data/lib/karafka/persistence/topic.rb +29 -0
  44. data/lib/karafka/process.rb +5 -4
  45. data/lib/karafka/responders/builder.rb +15 -14
  46. data/lib/karafka/routing/builder.rb +1 -1
  47. data/lib/karafka/routing/consumer_mapper.rb +3 -2
  48. data/lib/karafka/routing/router.rb +1 -1
  49. data/lib/karafka/routing/topic.rb +5 -11
  50. data/lib/karafka/schemas/config.rb +3 -0
  51. data/lib/karafka/schemas/consumer_group.rb +15 -3
  52. data/lib/karafka/schemas/consumer_group_topic.rb +1 -1
  53. data/lib/karafka/server.rb +37 -5
  54. data/lib/karafka/setup/config.rb +47 -21
  55. data/lib/karafka/setup/configurators/base.rb +6 -12
  56. data/lib/karafka/setup/configurators/params.rb +25 -0
  57. data/lib/karafka/setup/configurators/water_drop.rb +6 -3
  58. data/lib/karafka/setup/dsl.rb +22 -0
  59. data/lib/karafka/templates/{application_controller.rb.example → application_consumer.rb.example} +2 -3
  60. data/lib/karafka/templates/karafka.rb.example +17 -4
  61. data/lib/karafka/version.rb +1 -1
  62. metadata +58 -24
  63. data/.github/ISSUE_TEMPLATE.md +0 -2
  64. data/lib/karafka/connection/processor.rb +0 -61
  65. data/lib/karafka/controllers/includer.rb +0 -51
  66. data/lib/karafka/controllers/responders.rb +0 -19
  67. data/lib/karafka/logger.rb +0 -53
  68. data/lib/karafka/monitor.rb +0 -98
  69. data/lib/karafka/persistence/controller.rb +0 -38
@@ -3,10 +3,13 @@
3
3
  module Karafka
4
4
  module Setup
5
5
  # Configurators module is used to enclose all the external dependencies configurations
6
+ # upon which Karafka depents
6
7
  class Configurators
7
- # Karafka has come components that it relies on (like Sidekiq)
8
+ # Karafka has some components that it relies on (like Sidekiq)
8
9
  # We need to configure all of them only when the framework was set up.
9
10
  # Any class that descends from this one will be automatically invoked upon setup (after it)
11
+ # @note This should be used only for internal Karafka dependencies configuration
12
+ # End users configuration should go to the after_init block
10
13
  # @example Configure an Example class
11
14
  # class ExampleConfigurator < Base
12
15
  # def setup
@@ -15,18 +18,9 @@ module Karafka
15
18
  # end
16
19
  # end
17
20
  class Base
18
- extend ActiveSupport::DescendantsTracker
19
-
20
- attr_reader :config
21
-
22
- # @param config [Karafka::Config] config instance
23
- # @return [Karafka::Config::Base] configurator for a given component
24
- def initialize(config)
25
- @config = config
26
- end
27
-
21
+ # @param _config [Karafka::Config] config instance
28
22
  # This method needs to be implemented in a subclass
29
- def setup
23
+ def self.setup(_config)
30
24
  raise NotImplementedError
31
25
  end
32
26
  end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Setup
5
+ class Configurators
6
+ # Karafka::Params::Params are dynamically built based on user defined parent class
7
+ # so we cannot just require it, we need to initialize it after user is done with
8
+ # the framework configuration. This is a configurator that does exactly that.
9
+ class Params < Base
10
+ # Builds up Karafka::Params::Params class with user defined parent class
11
+ # @param config [Karafka::Setup::Config] Config we can user to setup things
12
+ def self.setup(config)
13
+ return if defined? Karafka::Params::Params
14
+
15
+ Karafka::Params.const_set(
16
+ 'Params',
17
+ Class
18
+ .new(config.params_base_class)
19
+ .tap { |klass| klass.include(Karafka::Params::Dsl) }
20
+ )
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -6,17 +6,20 @@ module Karafka
6
6
  # Class responsible for setting up WaterDrop configuration
7
7
  class WaterDrop < Base
8
8
  # Sets up a WaterDrop settings
9
- def setup
9
+ # @param config [Karafka::Setup::Config] Config we can user to setup things
10
+ # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
11
+ # so we have the same monitor within whole Karafka framework (same with logger)
12
+ def self.setup(config)
10
13
  ::WaterDrop.setup do |water_config|
11
14
  water_config.deliver = true
12
15
 
13
- Karafka::App.config.to_h.except(:kafka).each do |k, v|
16
+ config.to_h.except(:kafka).each do |k, v|
14
17
  key_assignment = :"#{k}="
15
18
  next unless water_config.respond_to?(key_assignment)
16
19
  water_config.public_send(key_assignment, v)
17
20
  end
18
21
 
19
- Karafka::App.config.kafka.to_h.each do |k, v|
22
+ config.kafka.to_h.each do |k, v|
20
23
  key_assignment = :"#{k}="
21
24
  next unless water_config.kafka.respond_to?(key_assignment)
22
25
  water_config.kafka.public_send(key_assignment, v)
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Setup
5
+ # Dsl for allowing to work with the configuration from the Karafka::App
6
+ # @note Despite providing methods, everything is still persisted and fetched
7
+ # from the Karafka::Setup::Config
8
+ module Dsl
9
+ # Sets up the whole configuration
10
+ # @param [Block] block configuration block
11
+ def setup(&block)
12
+ Setup::Config.setup(&block)
13
+ initialize!
14
+ end
15
+
16
+ # @return [Karafka::Config] config instance
17
+ def config
18
+ Setup::Config.config
19
+ end
20
+ end
21
+ end
22
+ end
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Application controller from which all Karafka controllers should inherit
3
+ # Application consumer from which all Karafka consumers should inherit
4
4
  # You can rename it if it would conflict with your current code base (in case you're integrating
5
5
  # Karafka with other frameworks)
6
- class ApplicationController < Karafka::BaseController
7
- end
6
+ ApplicationConsumer = Class.new(Karafka::BaseConsumer)
@@ -15,24 +15,37 @@ Karafka::Loader.load(Karafka::App.root)
15
15
 
16
16
  class KarafkaApp < Karafka::App
17
17
  setup do |config|
18
- config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
18
+ config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
19
19
  config.client_id = 'example_app'
20
20
  config.backend = :inline
21
21
  config.batch_fetching = true
22
+ # Uncomment this for Rails app integration
23
+ # config.logger = Rails.logger
22
24
  end
23
25
 
26
+ after_init do |config|
27
+ # Put here all the things you want to do after the Karafka framework
28
+ # initialization
29
+ end
30
+
31
+ # Comment out this part if you are not using instrumentation and/or you are not
32
+ # interested in logging events for certain environments. Since instrumentation
33
+ # notifications add extra boilerplate, if you want to achieve max performance,
34
+ # listen to only what you really need for given environment.
35
+ Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
36
+
24
37
  consumer_groups.draw do
25
38
  # topic :example do
26
- # controller ExampleController
39
+ # consumer ExampleConsumer
27
40
  # end
28
41
 
29
42
  # consumer_group :bigger_group do
30
43
  # topic :test do
31
- # controller TestController
44
+ # consumer TestConsumer
32
45
  # end
33
46
  #
34
47
  # topic :test2 do
35
- # controller Test2Controller
48
+ # consumer Test2Consumer
36
49
  # end
37
50
  # end
38
51
  end
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '1.1.0'
6
+ VERSION = '1.2.0'
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-11-07 00:00:00.000000000 Z
13
+ date: 2018-03-23 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport
@@ -18,14 +18,14 @@ dependencies:
18
18
  requirements:
19
19
  - - ">="
20
20
  - !ruby/object:Gem::Version
21
- version: '5.0'
21
+ version: '4.0'
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  requirements:
26
26
  - - ">="
27
27
  - !ruby/object:Gem::Version
28
- version: '5.0'
28
+ version: '4.0'
29
29
  - !ruby/object:Gem::Dependency
30
30
  name: dry-configurable
31
31
  requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,34 @@ dependencies:
40
40
  - - "~>"
41
41
  - !ruby/object:Gem::Version
42
42
  version: '0.7'
43
+ - !ruby/object:Gem::Dependency
44
+ name: dry-inflector
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: 0.1.1
50
+ type: :runtime
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - "~>"
55
+ - !ruby/object:Gem::Version
56
+ version: 0.1.1
57
+ - !ruby/object:Gem::Dependency
58
+ name: dry-monitor
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - "~>"
62
+ - !ruby/object:Gem::Version
63
+ version: '0.1'
64
+ type: :runtime
65
+ prerelease: false
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - "~>"
69
+ - !ruby/object:Gem::Version
70
+ version: '0.1'
43
71
  - !ruby/object:Gem::Dependency
44
72
  name: dry-validation
45
73
  requirement: !ruby/object:Gem::Requirement
@@ -116,14 +144,14 @@ dependencies:
116
144
  requirements:
117
145
  - - ">="
118
146
  - !ruby/object:Gem::Version
119
- version: '0.5'
147
+ version: 0.5.3
120
148
  type: :runtime
121
149
  prerelease: false
122
150
  version_requirements: !ruby/object:Gem::Requirement
123
151
  requirements:
124
152
  - - ">="
125
153
  - !ruby/object:Gem::Version
126
- version: '0.5'
154
+ version: 0.5.3
127
155
  - !ruby/object:Gem::Dependency
128
156
  name: thor
129
157
  requirement: !ruby/object:Gem::Requirement
@@ -142,16 +170,16 @@ dependencies:
142
170
  name: waterdrop
143
171
  requirement: !ruby/object:Gem::Requirement
144
172
  requirements:
145
- - - ">="
173
+ - - "~>"
146
174
  - !ruby/object:Gem::Version
147
- version: 1.0.1
175
+ version: '1.2'
148
176
  type: :runtime
149
177
  prerelease: false
150
178
  version_requirements: !ruby/object:Gem::Requirement
151
179
  requirements:
152
- - - ">="
180
+ - - "~>"
153
181
  - !ruby/object:Gem::Version
154
- version: 1.0.1
182
+ version: '1.2'
155
183
  description: Framework used to simplify Apache Kafka based Ruby applications development
156
184
  email:
157
185
  - maciej@coditsu.io
@@ -163,7 +191,6 @@ extensions: []
163
191
  extra_rdoc_files: []
164
192
  files:
165
193
  - ".console_irbrc"
166
- - ".github/ISSUE_TEMPLATE.md"
167
194
  - ".gitignore"
168
195
  - ".rspec"
169
196
  - ".ruby-gemset"
@@ -183,8 +210,11 @@ files:
183
210
  - lib/karafka/app.rb
184
211
  - lib/karafka/attributes_map.rb
185
212
  - lib/karafka/backends/inline.rb
186
- - lib/karafka/base_controller.rb
213
+ - lib/karafka/base_consumer.rb
187
214
  - lib/karafka/base_responder.rb
215
+ - lib/karafka/callbacks.rb
216
+ - lib/karafka/callbacks/config.rb
217
+ - lib/karafka/callbacks/dsl.rb
188
218
  - lib/karafka/cli.rb
189
219
  - lib/karafka/cli/base.rb
190
220
  - lib/karafka/cli/console.rb
@@ -192,29 +222,31 @@ files:
192
222
  - lib/karafka/cli/info.rb
193
223
  - lib/karafka/cli/install.rb
194
224
  - lib/karafka/cli/server.rb
225
+ - lib/karafka/connection/client.rb
195
226
  - lib/karafka/connection/config_adapter.rb
196
- - lib/karafka/connection/consumer.rb
227
+ - lib/karafka/connection/delegator.rb
197
228
  - lib/karafka/connection/listener.rb
198
- - lib/karafka/connection/processor.rb
199
- - lib/karafka/controllers/callbacks.rb
200
- - lib/karafka/controllers/includer.rb
201
- - lib/karafka/controllers/responders.rb
202
- - lib/karafka/controllers/single_params.rb
229
+ - lib/karafka/consumers/callbacks.rb
230
+ - lib/karafka/consumers/includer.rb
231
+ - lib/karafka/consumers/responders.rb
232
+ - lib/karafka/consumers/single_params.rb
203
233
  - lib/karafka/errors.rb
204
234
  - lib/karafka/fetcher.rb
205
235
  - lib/karafka/helpers/class_matcher.rb
206
236
  - lib/karafka/helpers/config_retriever.rb
207
237
  - lib/karafka/helpers/multi_delegator.rb
238
+ - lib/karafka/instrumentation/listener.rb
239
+ - lib/karafka/instrumentation/logger.rb
240
+ - lib/karafka/instrumentation/monitor.rb
208
241
  - lib/karafka/loader.rb
209
- - lib/karafka/logger.rb
210
- - lib/karafka/monitor.rb
211
- - lib/karafka/params/params.rb
242
+ - lib/karafka/params/dsl.rb
212
243
  - lib/karafka/params/params_batch.rb
213
244
  - lib/karafka/parsers/json.rb
214
245
  - lib/karafka/patches/dry_configurable.rb
215
246
  - lib/karafka/patches/ruby_kafka.rb
247
+ - lib/karafka/persistence/client.rb
216
248
  - lib/karafka/persistence/consumer.rb
217
- - lib/karafka/persistence/controller.rb
249
+ - lib/karafka/persistence/topic.rb
218
250
  - lib/karafka/process.rb
219
251
  - lib/karafka/responders/builder.rb
220
252
  - lib/karafka/responders/topic.rb
@@ -233,9 +265,11 @@ files:
233
265
  - lib/karafka/server.rb
234
266
  - lib/karafka/setup/config.rb
235
267
  - lib/karafka/setup/configurators/base.rb
268
+ - lib/karafka/setup/configurators/params.rb
236
269
  - lib/karafka/setup/configurators/water_drop.rb
270
+ - lib/karafka/setup/dsl.rb
237
271
  - lib/karafka/status.rb
238
- - lib/karafka/templates/application_controller.rb.example
272
+ - lib/karafka/templates/application_consumer.rb.example
239
273
  - lib/karafka/templates/application_responder.rb.example
240
274
  - lib/karafka/templates/karafka.rb.example
241
275
  - lib/karafka/version.rb
@@ -260,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
260
294
  version: '0'
261
295
  requirements: []
262
296
  rubyforge_project:
263
- rubygems_version: 2.6.13
297
+ rubygems_version: 2.7.6
264
298
  signing_key:
265
299
  specification_version: 4
266
300
  summary: Ruby based framework for working with Apache Kafka
@@ -1,2 +0,0 @@
1
- <!-- Love karafka? Please consider supporting our collective:
2
- 👉 https://opencollective.com/karafka/donate -->
@@ -1,61 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Connection
5
- # Class that consumes messages for which we listen
6
- module Processor
7
- class << self
8
- # Processes messages (does something with them)
9
- # It will either schedule or run a proper controller action for messages
10
- # @note This should be looped to obtain a constant listening
11
- # @note We catch all the errors here, to make sure that none failures
12
- # for a given consumption will affect other consumed messages
13
- # If we wouldn't catch it, it would propagate up until killing the thread
14
- # @param group_id [String] group_id of a group from which a given message came
15
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
16
- def process(group_id, kafka_messages)
17
- # @note We always get messages by topic and partition so we can take topic from the
18
- # first one and it will be valid for all the messages
19
- # We map from incoming topic name, as it might be namespaced, etc.
20
- # @see topic_mapper internal docs
21
- mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
22
- topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
23
- controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
24
- topic.controller.new
25
- end
26
-
27
- # Depending on a case (persisted or not) we might use new controller instance per each
28
- # batch, or use the same instance for all of them (for implementing buffering, etc)
29
- send(
30
- topic.batch_consuming ? :process_batch : :process_each,
31
- controller,
32
- kafka_messages
33
- )
34
- end
35
-
36
- private
37
-
38
- # Processes whole batch in one request (all at once)
39
- # @param controller [Karafka::BaseController] base controller descendant
40
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
41
- def process_batch(controller, kafka_messages)
42
- controller.params_batch = kafka_messages
43
- Karafka.monitor.notice(self, kafka_messages)
44
- controller.call
45
- end
46
-
47
- # Processes messages one by one (like with std http requests)
48
- # @param controller [Karafka::BaseController] base controller descendant
49
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
50
- def process_each(controller, kafka_messages)
51
- kafka_messages.each do |kafka_message|
52
- # @note This is a simple trick - we just process one after another, but in order
53
- # not to handle everywhere both cases (single vs batch), we just "fake" batching with
54
- # a single message for each
55
- process_batch(controller, [kafka_message])
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
@@ -1,51 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Additional functionalities for controllers
5
- module Controllers
6
- # Module used to inject functionalities into a given controller class, based on the controller
7
- # topic and its settings
8
- # We don't need all the behaviors in all the cases, so it is totally not worth having
9
- # everything in all the cases all the time
10
- module Includer
11
- class << self
12
- # @param controller_class [Class] controller class, that will get some functionalities
13
- # based on the topic under which it operates
14
- def call(controller_class)
15
- topic = controller_class.topic
16
-
17
- bind_backend(controller_class, topic)
18
- bind_params(controller_class, topic)
19
- bind_responders(controller_class, topic)
20
- end
21
-
22
- private
23
-
24
- # Figures out backend for a given controller class, based on the topic backend and
25
- # includes it into the controller class
26
- # @param controller_class [Class] controller class
27
- # @param topic [Karafka::Routing::Topic] topic of a controller class
28
- def bind_backend(controller_class, topic)
29
- backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
30
- controller_class.include backend
31
- end
32
-
33
- # Adds a single #params support for non batch processed topics
34
- # @param controller_class [Class] controller class
35
- # @param topic [Karafka::Routing::Topic] topic of a controller class
36
- def bind_params(controller_class, topic)
37
- return if topic.batch_consuming
38
- controller_class.include SingleParams
39
- end
40
-
41
- # Adds responders support for topics and controllers with responders defined for them
42
- # @param controller_class [Class] controller class
43
- # @param topic [Karafka::Routing::Topic] topic of a controller class
44
- def bind_responders(controller_class, topic)
45
- return unless topic.responder
46
- controller_class.include Responders
47
- end
48
- end
49
- end
50
- end
51
- end