karafka 1.1.1 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/.ruby-version +1 -1
  3. data/.travis.yml +1 -0
  4. data/CHANGELOG.md +40 -0
  5. data/Gemfile +1 -2
  6. data/Gemfile.lock +41 -29
  7. data/README.md +7 -4
  8. data/karafka.gemspec +6 -4
  9. data/lib/karafka.rb +17 -7
  10. data/lib/karafka/app.rb +8 -15
  11. data/lib/karafka/attributes_map.rb +1 -1
  12. data/lib/karafka/backends/inline.rb +1 -2
  13. data/lib/karafka/{base_controller.rb → base_consumer.rb} +19 -11
  14. data/lib/karafka/base_responder.rb +33 -14
  15. data/lib/karafka/callbacks.rb +30 -0
  16. data/lib/karafka/callbacks/config.rb +22 -0
  17. data/lib/karafka/callbacks/dsl.rb +16 -0
  18. data/lib/karafka/cli/install.rb +2 -3
  19. data/lib/karafka/cli/server.rb +0 -1
  20. data/lib/karafka/connection/{consumer.rb → client.rb} +32 -36
  21. data/lib/karafka/connection/config_adapter.rb +14 -6
  22. data/lib/karafka/connection/delegator.rb +46 -0
  23. data/lib/karafka/connection/listener.rb +22 -13
  24. data/lib/karafka/{controllers → consumers}/callbacks.rb +9 -9
  25. data/lib/karafka/consumers/includer.rb +51 -0
  26. data/lib/karafka/consumers/responders.rb +24 -0
  27. data/lib/karafka/{controllers → consumers}/single_params.rb +3 -3
  28. data/lib/karafka/errors.rb +10 -3
  29. data/lib/karafka/fetcher.rb +30 -34
  30. data/lib/karafka/helpers/class_matcher.rb +8 -8
  31. data/lib/karafka/helpers/config_retriever.rb +2 -2
  32. data/lib/karafka/instrumentation/listener.rb +112 -0
  33. data/lib/karafka/instrumentation/logger.rb +55 -0
  34. data/lib/karafka/instrumentation/monitor.rb +64 -0
  35. data/lib/karafka/loader.rb +0 -1
  36. data/lib/karafka/params/{params.rb → dsl.rb} +71 -43
  37. data/lib/karafka/params/params_batch.rb +7 -2
  38. data/lib/karafka/patches/dry_configurable.rb +6 -2
  39. data/lib/karafka/patches/ruby_kafka.rb +10 -10
  40. data/lib/karafka/persistence/client.rb +25 -0
  41. data/lib/karafka/persistence/consumer.rb +27 -14
  42. data/lib/karafka/persistence/topic.rb +29 -0
  43. data/lib/karafka/process.rb +5 -4
  44. data/lib/karafka/responders/builder.rb +15 -14
  45. data/lib/karafka/routing/builder.rb +1 -1
  46. data/lib/karafka/routing/consumer_mapper.rb +3 -2
  47. data/lib/karafka/routing/router.rb +1 -1
  48. data/lib/karafka/routing/topic.rb +5 -5
  49. data/lib/karafka/schemas/config.rb +3 -0
  50. data/lib/karafka/schemas/consumer_group.rb +15 -3
  51. data/lib/karafka/schemas/consumer_group_topic.rb +1 -1
  52. data/lib/karafka/server.rb +37 -5
  53. data/lib/karafka/setup/config.rb +45 -21
  54. data/lib/karafka/setup/configurators/base.rb +6 -12
  55. data/lib/karafka/setup/configurators/params.rb +25 -0
  56. data/lib/karafka/setup/configurators/water_drop.rb +6 -3
  57. data/lib/karafka/setup/dsl.rb +22 -0
  58. data/lib/karafka/templates/{application_controller.rb.example → application_consumer.rb.example} +2 -3
  59. data/lib/karafka/templates/karafka.rb.example +17 -4
  60. data/lib/karafka/version.rb +1 -1
  61. metadata +58 -23
  62. data/lib/karafka/connection/processor.rb +0 -61
  63. data/lib/karafka/controllers/includer.rb +0 -51
  64. data/lib/karafka/controllers/responders.rb +0 -19
  65. data/lib/karafka/logger.rb +0 -53
  66. data/lib/karafka/monitor.rb +0 -98
  67. data/lib/karafka/persistence/controller.rb +0 -38
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Setup
5
+ class Configurators
6
+ # Karafka::Params::Params are dynamically built based on user defined parent class
7
+ # so we cannot just require it, we need to initialize it after user is done with
8
+ # the framework configuration. This is a configurator that does exactly that.
9
+ class Params < Base
10
+ # Builds up Karafka::Params::Params class with user defined parent class
11
+ # @param config [Karafka::Setup::Config] Config we can user to setup things
12
+ def self.setup(config)
13
+ return if defined? Karafka::Params::Params
14
+
15
+ Karafka::Params.const_set(
16
+ 'Params',
17
+ Class
18
+ .new(config.params_base_class)
19
+ .tap { |klass| klass.include(Karafka::Params::Dsl) }
20
+ )
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -6,17 +6,20 @@ module Karafka
6
6
  # Class responsible for setting up WaterDrop configuration
7
7
  class WaterDrop < Base
8
8
  # Sets up a WaterDrop settings
9
- def setup
9
+ # @param config [Karafka::Setup::Config] Config we can user to setup things
10
+ # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
11
+ # so we have the same monitor within whole Karafka framework (same with logger)
12
+ def self.setup(config)
10
13
  ::WaterDrop.setup do |water_config|
11
14
  water_config.deliver = true
12
15
 
13
- Karafka::App.config.to_h.except(:kafka).each do |k, v|
16
+ config.to_h.except(:kafka).each do |k, v|
14
17
  key_assignment = :"#{k}="
15
18
  next unless water_config.respond_to?(key_assignment)
16
19
  water_config.public_send(key_assignment, v)
17
20
  end
18
21
 
19
- Karafka::App.config.kafka.to_h.each do |k, v|
22
+ config.kafka.to_h.each do |k, v|
20
23
  key_assignment = :"#{k}="
21
24
  next unless water_config.kafka.respond_to?(key_assignment)
22
25
  water_config.kafka.public_send(key_assignment, v)
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Setup
5
+ # Dsl for allowing to work with the configuration from the Karafka::App
6
+ # @note Despite providing methods, everything is still persisted and fetched
7
+ # from the Karafka::Setup::Config
8
+ module Dsl
9
+ # Sets up the whole configuration
10
+ # @param [Block] block configuration block
11
+ def setup(&block)
12
+ Setup::Config.setup(&block)
13
+ initialize!
14
+ end
15
+
16
+ # @return [Karafka::Config] config instance
17
+ def config
18
+ Setup::Config.config
19
+ end
20
+ end
21
+ end
22
+ end
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Application controller from which all Karafka controllers should inherit
3
+ # Application consumer from which all Karafka consumers should inherit
4
4
  # You can rename it if it would conflict with your current code base (in case you're integrating
5
5
  # Karafka with other frameworks)
6
- class ApplicationController < Karafka::BaseController
7
- end
6
+ ApplicationConsumer = Class.new(Karafka::BaseConsumer)
@@ -15,24 +15,37 @@ Karafka::Loader.load(Karafka::App.root)
15
15
 
16
16
  class KarafkaApp < Karafka::App
17
17
  setup do |config|
18
- config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
18
+ config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
19
19
  config.client_id = 'example_app'
20
20
  config.backend = :inline
21
21
  config.batch_fetching = true
22
+ # Uncomment this for Rails app integration
23
+ # config.logger = Rails.logger
22
24
  end
23
25
 
26
+ after_init do |config|
27
+ # Put here all the things you want to do after the Karafka framework
28
+ # initialization
29
+ end
30
+
31
+ # Comment out this part if you are not using instrumentation and/or you are not
32
+ # interested in logging events for certain environments. Since instrumentation
33
+ # notifications add extra boilerplate, if you want to achieve max performance,
34
+ # listen to only what you really need for given environment.
35
+ Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
36
+
24
37
  consumer_groups.draw do
25
38
  # topic :example do
26
- # controller ExampleController
39
+ # consumer ExampleConsumer
27
40
  # end
28
41
 
29
42
  # consumer_group :bigger_group do
30
43
  # topic :test do
31
- # controller TestController
44
+ # consumer TestConsumer
32
45
  # end
33
46
  #
34
47
  # topic :test2 do
35
- # controller Test2Controller
48
+ # consumer Test2Consumer
36
49
  # end
37
50
  # end
38
51
  end
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '1.1.1'
6
+ VERSION = '1.2.0'
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-11-20 00:00:00.000000000 Z
13
+ date: 2018-03-23 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport
@@ -18,14 +18,14 @@ dependencies:
18
18
  requirements:
19
19
  - - ">="
20
20
  - !ruby/object:Gem::Version
21
- version: '5.0'
21
+ version: '4.0'
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  requirements:
26
26
  - - ">="
27
27
  - !ruby/object:Gem::Version
28
- version: '5.0'
28
+ version: '4.0'
29
29
  - !ruby/object:Gem::Dependency
30
30
  name: dry-configurable
31
31
  requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,34 @@ dependencies:
40
40
  - - "~>"
41
41
  - !ruby/object:Gem::Version
42
42
  version: '0.7'
43
+ - !ruby/object:Gem::Dependency
44
+ name: dry-inflector
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: 0.1.1
50
+ type: :runtime
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - "~>"
55
+ - !ruby/object:Gem::Version
56
+ version: 0.1.1
57
+ - !ruby/object:Gem::Dependency
58
+ name: dry-monitor
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - "~>"
62
+ - !ruby/object:Gem::Version
63
+ version: '0.1'
64
+ type: :runtime
65
+ prerelease: false
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - "~>"
69
+ - !ruby/object:Gem::Version
70
+ version: '0.1'
43
71
  - !ruby/object:Gem::Dependency
44
72
  name: dry-validation
45
73
  requirement: !ruby/object:Gem::Requirement
@@ -116,14 +144,14 @@ dependencies:
116
144
  requirements:
117
145
  - - ">="
118
146
  - !ruby/object:Gem::Version
119
- version: '0.5'
147
+ version: 0.5.3
120
148
  type: :runtime
121
149
  prerelease: false
122
150
  version_requirements: !ruby/object:Gem::Requirement
123
151
  requirements:
124
152
  - - ">="
125
153
  - !ruby/object:Gem::Version
126
- version: '0.5'
154
+ version: 0.5.3
127
155
  - !ruby/object:Gem::Dependency
128
156
  name: thor
129
157
  requirement: !ruby/object:Gem::Requirement
@@ -142,16 +170,16 @@ dependencies:
142
170
  name: waterdrop
143
171
  requirement: !ruby/object:Gem::Requirement
144
172
  requirements:
145
- - - ">="
173
+ - - "~>"
146
174
  - !ruby/object:Gem::Version
147
- version: 1.0.1
175
+ version: '1.2'
148
176
  type: :runtime
149
177
  prerelease: false
150
178
  version_requirements: !ruby/object:Gem::Requirement
151
179
  requirements:
152
- - - ">="
180
+ - - "~>"
153
181
  - !ruby/object:Gem::Version
154
- version: 1.0.1
182
+ version: '1.2'
155
183
  description: Framework used to simplify Apache Kafka based Ruby applications development
156
184
  email:
157
185
  - maciej@coditsu.io
@@ -182,8 +210,11 @@ files:
182
210
  - lib/karafka/app.rb
183
211
  - lib/karafka/attributes_map.rb
184
212
  - lib/karafka/backends/inline.rb
185
- - lib/karafka/base_controller.rb
213
+ - lib/karafka/base_consumer.rb
186
214
  - lib/karafka/base_responder.rb
215
+ - lib/karafka/callbacks.rb
216
+ - lib/karafka/callbacks/config.rb
217
+ - lib/karafka/callbacks/dsl.rb
187
218
  - lib/karafka/cli.rb
188
219
  - lib/karafka/cli/base.rb
189
220
  - lib/karafka/cli/console.rb
@@ -191,29 +222,31 @@ files:
191
222
  - lib/karafka/cli/info.rb
192
223
  - lib/karafka/cli/install.rb
193
224
  - lib/karafka/cli/server.rb
225
+ - lib/karafka/connection/client.rb
194
226
  - lib/karafka/connection/config_adapter.rb
195
- - lib/karafka/connection/consumer.rb
227
+ - lib/karafka/connection/delegator.rb
196
228
  - lib/karafka/connection/listener.rb
197
- - lib/karafka/connection/processor.rb
198
- - lib/karafka/controllers/callbacks.rb
199
- - lib/karafka/controllers/includer.rb
200
- - lib/karafka/controllers/responders.rb
201
- - lib/karafka/controllers/single_params.rb
229
+ - lib/karafka/consumers/callbacks.rb
230
+ - lib/karafka/consumers/includer.rb
231
+ - lib/karafka/consumers/responders.rb
232
+ - lib/karafka/consumers/single_params.rb
202
233
  - lib/karafka/errors.rb
203
234
  - lib/karafka/fetcher.rb
204
235
  - lib/karafka/helpers/class_matcher.rb
205
236
  - lib/karafka/helpers/config_retriever.rb
206
237
  - lib/karafka/helpers/multi_delegator.rb
238
+ - lib/karafka/instrumentation/listener.rb
239
+ - lib/karafka/instrumentation/logger.rb
240
+ - lib/karafka/instrumentation/monitor.rb
207
241
  - lib/karafka/loader.rb
208
- - lib/karafka/logger.rb
209
- - lib/karafka/monitor.rb
210
- - lib/karafka/params/params.rb
242
+ - lib/karafka/params/dsl.rb
211
243
  - lib/karafka/params/params_batch.rb
212
244
  - lib/karafka/parsers/json.rb
213
245
  - lib/karafka/patches/dry_configurable.rb
214
246
  - lib/karafka/patches/ruby_kafka.rb
247
+ - lib/karafka/persistence/client.rb
215
248
  - lib/karafka/persistence/consumer.rb
216
- - lib/karafka/persistence/controller.rb
249
+ - lib/karafka/persistence/topic.rb
217
250
  - lib/karafka/process.rb
218
251
  - lib/karafka/responders/builder.rb
219
252
  - lib/karafka/responders/topic.rb
@@ -232,9 +265,11 @@ files:
232
265
  - lib/karafka/server.rb
233
266
  - lib/karafka/setup/config.rb
234
267
  - lib/karafka/setup/configurators/base.rb
268
+ - lib/karafka/setup/configurators/params.rb
235
269
  - lib/karafka/setup/configurators/water_drop.rb
270
+ - lib/karafka/setup/dsl.rb
236
271
  - lib/karafka/status.rb
237
- - lib/karafka/templates/application_controller.rb.example
272
+ - lib/karafka/templates/application_consumer.rb.example
238
273
  - lib/karafka/templates/application_responder.rb.example
239
274
  - lib/karafka/templates/karafka.rb.example
240
275
  - lib/karafka/version.rb
@@ -259,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
259
294
  version: '0'
260
295
  requirements: []
261
296
  rubyforge_project:
262
- rubygems_version: 2.6.13
297
+ rubygems_version: 2.7.6
263
298
  signing_key:
264
299
  specification_version: 4
265
300
  summary: Ruby based framework for working with Apache Kafka
@@ -1,61 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Connection
5
- # Class that consumes messages for which we listen
6
- module Processor
7
- class << self
8
- # Processes messages (does something with them)
9
- # It will either schedule or run a proper controller action for messages
10
- # @note This should be looped to obtain a constant listening
11
- # @note We catch all the errors here, to make sure that none failures
12
- # for a given consumption will affect other consumed messages
13
- # If we wouldn't catch it, it would propagate up until killing the thread
14
- # @param group_id [String] group_id of a group from which a given message came
15
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
16
- def process(group_id, kafka_messages)
17
- # @note We always get messages by topic and partition so we can take topic from the
18
- # first one and it will be valid for all the messages
19
- # We map from incoming topic name, as it might be namespaced, etc.
20
- # @see topic_mapper internal docs
21
- mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
22
- topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
23
- controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
24
- topic.controller.new
25
- end
26
-
27
- # Depending on a case (persisted or not) we might use new controller instance per each
28
- # batch, or use the same instance for all of them (for implementing buffering, etc)
29
- send(
30
- topic.batch_consuming ? :process_batch : :process_each,
31
- controller,
32
- kafka_messages
33
- )
34
- end
35
-
36
- private
37
-
38
- # Processes whole batch in one request (all at once)
39
- # @param controller [Karafka::BaseController] base controller descendant
40
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
41
- def process_batch(controller, kafka_messages)
42
- controller.params_batch = kafka_messages
43
- Karafka.monitor.notice(self, kafka_messages)
44
- controller.call
45
- end
46
-
47
- # Processes messages one by one (like with std http requests)
48
- # @param controller [Karafka::BaseController] base controller descendant
49
- # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
50
- def process_each(controller, kafka_messages)
51
- kafka_messages.each do |kafka_message|
52
- # @note This is a simple trick - we just process one after another, but in order
53
- # not to handle everywhere both cases (single vs batch), we just "fake" batching with
54
- # a single message for each
55
- process_batch(controller, [kafka_message])
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
@@ -1,51 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Additional functionalities for controllers
5
- module Controllers
6
- # Module used to inject functionalities into a given controller class, based on the controller
7
- # topic and its settings
8
- # We don't need all the behaviors in all the cases, so it is totally not worth having
9
- # everything in all the cases all the time
10
- module Includer
11
- class << self
12
- # @param controller_class [Class] controller class, that will get some functionalities
13
- # based on the topic under which it operates
14
- def call(controller_class)
15
- topic = controller_class.topic
16
-
17
- bind_backend(controller_class, topic)
18
- bind_params(controller_class, topic)
19
- bind_responders(controller_class, topic)
20
- end
21
-
22
- private
23
-
24
- # Figures out backend for a given controller class, based on the topic backend and
25
- # includes it into the controller class
26
- # @param controller_class [Class] controller class
27
- # @param topic [Karafka::Routing::Topic] topic of a controller class
28
- def bind_backend(controller_class, topic)
29
- backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
30
- controller_class.include backend
31
- end
32
-
33
- # Adds a single #params support for non batch processed topics
34
- # @param controller_class [Class] controller class
35
- # @param topic [Karafka::Routing::Topic] topic of a controller class
36
- def bind_params(controller_class, topic)
37
- return if topic.batch_consuming
38
- controller_class.include SingleParams
39
- end
40
-
41
- # Adds responders support for topics and controllers with responders defined for them
42
- # @param controller_class [Class] controller class
43
- # @param topic [Karafka::Routing::Topic] topic of a controller class
44
- def bind_responders(controller_class, topic)
45
- return unless topic.responder
46
- controller_class.include Responders
47
- end
48
- end
49
- end
50
- end
51
- end
@@ -1,19 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Controllers
5
- # Feature that allows us to use responders flow in controller
6
- module Responders
7
- # Responds with given data using given responder. This allows us to have a similar way of
8
- # defining flows like synchronous protocols
9
- # @param data Anything we want to pass to responder based on which we want to trigger further
10
- # Kafka responding
11
- def respond_with(*data)
12
- Karafka.monitor.notice(self.class, data: data)
13
- # @note we build a new instance of responder each time, as a long running (persisted)
14
- # controllers can respond multiple times during the lifecycle
15
- topic.responder.new(topic.parser).call(*data)
16
- end
17
- end
18
- end
19
- end