karafka 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +7 -0
  2. data/.console_irbrc +13 -0
  3. data/.gitignore +68 -0
  4. data/.rspec +1 -0
  5. data/.ruby-gemset +1 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +18 -0
  8. data/CHANGELOG.md +415 -0
  9. data/CODE_OF_CONDUCT.md +46 -0
  10. data/CONTRIBUTING.md +41 -0
  11. data/Gemfile +11 -0
  12. data/Gemfile.lock +123 -0
  13. data/MIT-LICENCE +18 -0
  14. data/README.md +89 -0
  15. data/bin/karafka +19 -0
  16. data/config/errors.yml +6 -0
  17. data/karafka.gemspec +37 -0
  18. data/lib/karafka.rb +78 -0
  19. data/lib/karafka/app.rb +45 -0
  20. data/lib/karafka/attributes_map.rb +67 -0
  21. data/lib/karafka/backends/inline.rb +16 -0
  22. data/lib/karafka/base_consumer.rb +68 -0
  23. data/lib/karafka/base_responder.rb +204 -0
  24. data/lib/karafka/callbacks.rb +30 -0
  25. data/lib/karafka/callbacks/config.rb +22 -0
  26. data/lib/karafka/callbacks/dsl.rb +16 -0
  27. data/lib/karafka/cli.rb +54 -0
  28. data/lib/karafka/cli/base.rb +78 -0
  29. data/lib/karafka/cli/console.rb +29 -0
  30. data/lib/karafka/cli/flow.rb +46 -0
  31. data/lib/karafka/cli/info.rb +29 -0
  32. data/lib/karafka/cli/install.rb +42 -0
  33. data/lib/karafka/cli/server.rb +66 -0
  34. data/lib/karafka/connection/client.rb +117 -0
  35. data/lib/karafka/connection/config_adapter.rb +120 -0
  36. data/lib/karafka/connection/delegator.rb +46 -0
  37. data/lib/karafka/connection/listener.rb +60 -0
  38. data/lib/karafka/consumers/callbacks.rb +54 -0
  39. data/lib/karafka/consumers/includer.rb +51 -0
  40. data/lib/karafka/consumers/responders.rb +24 -0
  41. data/lib/karafka/consumers/single_params.rb +15 -0
  42. data/lib/karafka/errors.rb +50 -0
  43. data/lib/karafka/fetcher.rb +44 -0
  44. data/lib/karafka/helpers/class_matcher.rb +78 -0
  45. data/lib/karafka/helpers/config_retriever.rb +46 -0
  46. data/lib/karafka/helpers/multi_delegator.rb +33 -0
  47. data/lib/karafka/instrumentation/listener.rb +112 -0
  48. data/lib/karafka/instrumentation/logger.rb +55 -0
  49. data/lib/karafka/instrumentation/monitor.rb +64 -0
  50. data/lib/karafka/loader.rb +28 -0
  51. data/lib/karafka/params/dsl.rb +156 -0
  52. data/lib/karafka/params/params_batch.rb +46 -0
  53. data/lib/karafka/parsers/json.rb +38 -0
  54. data/lib/karafka/patches/dry_configurable.rb +35 -0
  55. data/lib/karafka/patches/ruby_kafka.rb +34 -0
  56. data/lib/karafka/persistence/client.rb +25 -0
  57. data/lib/karafka/persistence/consumer.rb +38 -0
  58. data/lib/karafka/persistence/topic.rb +29 -0
  59. data/lib/karafka/process.rb +64 -0
  60. data/lib/karafka/responders/builder.rb +36 -0
  61. data/lib/karafka/responders/topic.rb +57 -0
  62. data/lib/karafka/routing/builder.rb +61 -0
  63. data/lib/karafka/routing/consumer_group.rb +61 -0
  64. data/lib/karafka/routing/consumer_mapper.rb +34 -0
  65. data/lib/karafka/routing/proxy.rb +37 -0
  66. data/lib/karafka/routing/router.rb +29 -0
  67. data/lib/karafka/routing/topic.rb +60 -0
  68. data/lib/karafka/routing/topic_mapper.rb +55 -0
  69. data/lib/karafka/schemas/config.rb +24 -0
  70. data/lib/karafka/schemas/consumer_group.rb +77 -0
  71. data/lib/karafka/schemas/consumer_group_topic.rb +18 -0
  72. data/lib/karafka/schemas/responder_usage.rb +39 -0
  73. data/lib/karafka/schemas/server_cli_options.rb +43 -0
  74. data/lib/karafka/server.rb +94 -0
  75. data/lib/karafka/setup/config.rb +189 -0
  76. data/lib/karafka/setup/configurators/base.rb +29 -0
  77. data/lib/karafka/setup/configurators/params.rb +25 -0
  78. data/lib/karafka/setup/configurators/water_drop.rb +32 -0
  79. data/lib/karafka/setup/dsl.rb +22 -0
  80. data/lib/karafka/status.rb +25 -0
  81. data/lib/karafka/templates/application_consumer.rb.example +6 -0
  82. data/lib/karafka/templates/application_responder.rb.example +11 -0
  83. data/lib/karafka/templates/karafka.rb.example +54 -0
  84. data/lib/karafka/version.rb +7 -0
  85. data/log/.gitkeep +0 -0
  86. metadata +301 -0
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Application consumer from which all Karafka consumers should inherit
4
+ # You can rename it if it would conflict with your current code base (in case you're integrating
5
+ # Karafka with other frameworks)
6
+ ApplicationConsumer = Class.new(Karafka::BaseConsumer)
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Application responder from which all Karafka responders should inherit
4
+ # You can rename it if it would conflict with your current code base (in case you're integrating
5
+ # Karafka with other frameworks)
6
+ class ApplicationResponder < Karafka::BaseResponder
7
+ # This method needs to be implemented in each of the responders
8
+ # def respond(data)
9
+ # respond_to :topic, data.to_json
10
+ # end
11
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Non Ruby on Rails setup
4
+ ENV['RACK_ENV'] ||= 'development'
5
+ ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
6
+ Bundler.require(:default, ENV['KARAFKA_ENV'])
7
+ Karafka::Loader.load(Karafka::App.root)
8
+
9
+ # Ruby on Rails setup
10
+ # Remove whole non-Rails setup that is above and uncomment the 4 lines below
11
+ # ENV['RAILS_ENV'] ||= 'development'
12
+ # ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
13
+ # require ::File.expand_path('../config/environment', __FILE__)
14
+ # Rails.application.eager_load!
15
+
16
+ class KarafkaApp < Karafka::App
17
+ setup do |config|
18
+ config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
19
+ config.client_id = 'example_app'
20
+ config.backend = :inline
21
+ config.batch_fetching = true
22
+ # Uncomment this for Rails app integration
23
+ # config.logger = Rails.logger
24
+ end
25
+
26
+ after_init do |config|
27
+ # Put here all the things you want to do after the Karafka framework
28
+ # initialization
29
+ end
30
+
31
+ # Comment out this part if you are not using instrumentation and/or you are not
32
+ # interested in logging events for certain environments. Since instrumentation
33
+ # notifications add extra boilerplate, if you want to achieve max performance,
34
+ # listen to only what you really need for given environment.
35
+ Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
36
+
37
+ consumer_groups.draw do
38
+ # topic :example do
39
+ # consumer ExampleConsumer
40
+ # end
41
+
42
+ # consumer_group :bigger_group do
43
+ # topic :test do
44
+ # consumer TestConsumer
45
+ # end
46
+ #
47
+ # topic :test2 do
48
+ # consumer Test2Consumer
49
+ # end
50
+ # end
51
+ end
52
+ end
53
+
54
+ KarafkaApp.boot!
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Main module namespace
4
+ module Karafka
5
+ # Current Karafka version
6
+ VERSION = '1.2.0'
7
+ end
File without changes
metadata ADDED
@@ -0,0 +1,301 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: karafka
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Maciej Mensfeld
8
+ - Pavlo Vavruk
9
+ - Adam Gwozdowski
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2018-03-23 00:00:00.000000000 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: activesupport
17
+ requirement: !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - ">="
20
+ - !ruby/object:Gem::Version
21
+ version: '4.0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ version: '4.0'
29
+ - !ruby/object:Gem::Dependency
30
+ name: dry-configurable
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - "~>"
34
+ - !ruby/object:Gem::Version
35
+ version: '0.7'
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - "~>"
41
+ - !ruby/object:Gem::Version
42
+ version: '0.7'
43
+ - !ruby/object:Gem::Dependency
44
+ name: dry-inflector
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: 0.1.1
50
+ type: :runtime
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - "~>"
55
+ - !ruby/object:Gem::Version
56
+ version: 0.1.1
57
+ - !ruby/object:Gem::Dependency
58
+ name: dry-monitor
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - "~>"
62
+ - !ruby/object:Gem::Version
63
+ version: '0.1'
64
+ type: :runtime
65
+ prerelease: false
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - "~>"
69
+ - !ruby/object:Gem::Version
70
+ version: '0.1'
71
+ - !ruby/object:Gem::Dependency
72
+ name: dry-validation
73
+ requirement: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - "~>"
76
+ - !ruby/object:Gem::Version
77
+ version: '0.11'
78
+ type: :runtime
79
+ prerelease: false
80
+ version_requirements: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - "~>"
83
+ - !ruby/object:Gem::Version
84
+ version: '0.11'
85
+ - !ruby/object:Gem::Dependency
86
+ name: envlogic
87
+ requirement: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - "~>"
90
+ - !ruby/object:Gem::Version
91
+ version: '1.0'
92
+ type: :runtime
93
+ prerelease: false
94
+ version_requirements: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - "~>"
97
+ - !ruby/object:Gem::Version
98
+ version: '1.0'
99
+ - !ruby/object:Gem::Dependency
100
+ name: multi_json
101
+ requirement: !ruby/object:Gem::Requirement
102
+ requirements:
103
+ - - ">="
104
+ - !ruby/object:Gem::Version
105
+ version: '1.12'
106
+ type: :runtime
107
+ prerelease: false
108
+ version_requirements: !ruby/object:Gem::Requirement
109
+ requirements:
110
+ - - ">="
111
+ - !ruby/object:Gem::Version
112
+ version: '1.12'
113
+ - !ruby/object:Gem::Dependency
114
+ name: rake
115
+ requirement: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - ">="
118
+ - !ruby/object:Gem::Version
119
+ version: '11.3'
120
+ type: :runtime
121
+ prerelease: false
122
+ version_requirements: !ruby/object:Gem::Requirement
123
+ requirements:
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ version: '11.3'
127
+ - !ruby/object:Gem::Dependency
128
+ name: require_all
129
+ requirement: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '1.4'
134
+ type: :runtime
135
+ prerelease: false
136
+ version_requirements: !ruby/object:Gem::Requirement
137
+ requirements:
138
+ - - ">="
139
+ - !ruby/object:Gem::Version
140
+ version: '1.4'
141
+ - !ruby/object:Gem::Dependency
142
+ name: ruby-kafka
143
+ requirement: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - ">="
146
+ - !ruby/object:Gem::Version
147
+ version: 0.5.3
148
+ type: :runtime
149
+ prerelease: false
150
+ version_requirements: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ version: 0.5.3
155
+ - !ruby/object:Gem::Dependency
156
+ name: thor
157
+ requirement: !ruby/object:Gem::Requirement
158
+ requirements:
159
+ - - "~>"
160
+ - !ruby/object:Gem::Version
161
+ version: '0.19'
162
+ type: :runtime
163
+ prerelease: false
164
+ version_requirements: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - "~>"
167
+ - !ruby/object:Gem::Version
168
+ version: '0.19'
169
+ - !ruby/object:Gem::Dependency
170
+ name: waterdrop
171
+ requirement: !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - "~>"
174
+ - !ruby/object:Gem::Version
175
+ version: '1.2'
176
+ type: :runtime
177
+ prerelease: false
178
+ version_requirements: !ruby/object:Gem::Requirement
179
+ requirements:
180
+ - - "~>"
181
+ - !ruby/object:Gem::Version
182
+ version: '1.2'
183
+ description: Framework used to simplify Apache Kafka based Ruby applications development
184
+ email:
185
+ - maciej@coditsu.io
186
+ - pavlo.vavruk@gmail.com
187
+ - adam99g@gmail.com
188
+ executables:
189
+ - karafka
190
+ extensions: []
191
+ extra_rdoc_files: []
192
+ files:
193
+ - ".console_irbrc"
194
+ - ".gitignore"
195
+ - ".rspec"
196
+ - ".ruby-gemset"
197
+ - ".ruby-version"
198
+ - ".travis.yml"
199
+ - CHANGELOG.md
200
+ - CODE_OF_CONDUCT.md
201
+ - CONTRIBUTING.md
202
+ - Gemfile
203
+ - Gemfile.lock
204
+ - MIT-LICENCE
205
+ - README.md
206
+ - bin/karafka
207
+ - config/errors.yml
208
+ - karafka.gemspec
209
+ - lib/karafka.rb
210
+ - lib/karafka/app.rb
211
+ - lib/karafka/attributes_map.rb
212
+ - lib/karafka/backends/inline.rb
213
+ - lib/karafka/base_consumer.rb
214
+ - lib/karafka/base_responder.rb
215
+ - lib/karafka/callbacks.rb
216
+ - lib/karafka/callbacks/config.rb
217
+ - lib/karafka/callbacks/dsl.rb
218
+ - lib/karafka/cli.rb
219
+ - lib/karafka/cli/base.rb
220
+ - lib/karafka/cli/console.rb
221
+ - lib/karafka/cli/flow.rb
222
+ - lib/karafka/cli/info.rb
223
+ - lib/karafka/cli/install.rb
224
+ - lib/karafka/cli/server.rb
225
+ - lib/karafka/connection/client.rb
226
+ - lib/karafka/connection/config_adapter.rb
227
+ - lib/karafka/connection/delegator.rb
228
+ - lib/karafka/connection/listener.rb
229
+ - lib/karafka/consumers/callbacks.rb
230
+ - lib/karafka/consumers/includer.rb
231
+ - lib/karafka/consumers/responders.rb
232
+ - lib/karafka/consumers/single_params.rb
233
+ - lib/karafka/errors.rb
234
+ - lib/karafka/fetcher.rb
235
+ - lib/karafka/helpers/class_matcher.rb
236
+ - lib/karafka/helpers/config_retriever.rb
237
+ - lib/karafka/helpers/multi_delegator.rb
238
+ - lib/karafka/instrumentation/listener.rb
239
+ - lib/karafka/instrumentation/logger.rb
240
+ - lib/karafka/instrumentation/monitor.rb
241
+ - lib/karafka/loader.rb
242
+ - lib/karafka/params/dsl.rb
243
+ - lib/karafka/params/params_batch.rb
244
+ - lib/karafka/parsers/json.rb
245
+ - lib/karafka/patches/dry_configurable.rb
246
+ - lib/karafka/patches/ruby_kafka.rb
247
+ - lib/karafka/persistence/client.rb
248
+ - lib/karafka/persistence/consumer.rb
249
+ - lib/karafka/persistence/topic.rb
250
+ - lib/karafka/process.rb
251
+ - lib/karafka/responders/builder.rb
252
+ - lib/karafka/responders/topic.rb
253
+ - lib/karafka/routing/builder.rb
254
+ - lib/karafka/routing/consumer_group.rb
255
+ - lib/karafka/routing/consumer_mapper.rb
256
+ - lib/karafka/routing/proxy.rb
257
+ - lib/karafka/routing/router.rb
258
+ - lib/karafka/routing/topic.rb
259
+ - lib/karafka/routing/topic_mapper.rb
260
+ - lib/karafka/schemas/config.rb
261
+ - lib/karafka/schemas/consumer_group.rb
262
+ - lib/karafka/schemas/consumer_group_topic.rb
263
+ - lib/karafka/schemas/responder_usage.rb
264
+ - lib/karafka/schemas/server_cli_options.rb
265
+ - lib/karafka/server.rb
266
+ - lib/karafka/setup/config.rb
267
+ - lib/karafka/setup/configurators/base.rb
268
+ - lib/karafka/setup/configurators/params.rb
269
+ - lib/karafka/setup/configurators/water_drop.rb
270
+ - lib/karafka/setup/dsl.rb
271
+ - lib/karafka/status.rb
272
+ - lib/karafka/templates/application_consumer.rb.example
273
+ - lib/karafka/templates/application_responder.rb.example
274
+ - lib/karafka/templates/karafka.rb.example
275
+ - lib/karafka/version.rb
276
+ - log/.gitkeep
277
+ homepage: https://github.com/karafka/karafka
278
+ licenses:
279
+ - MIT
280
+ metadata: {}
281
+ post_install_message:
282
+ rdoc_options: []
283
+ require_paths:
284
+ - lib
285
+ required_ruby_version: !ruby/object:Gem::Requirement
286
+ requirements:
287
+ - - ">="
288
+ - !ruby/object:Gem::Version
289
+ version: 2.3.0
290
+ required_rubygems_version: !ruby/object:Gem::Requirement
291
+ requirements:
292
+ - - ">="
293
+ - !ruby/object:Gem::Version
294
+ version: '0'
295
+ requirements: []
296
+ rubyforge_project:
297
+ rubygems_version: 2.7.6
298
+ signing_key:
299
+ specification_version: 4
300
+ summary: Ruby based framework for working with Apache Kafka
301
+ test_files: []