karafka 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +68 -0
  3. data/.ruby-gemset +1 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +6 -0
  6. data/CHANGELOG.md +202 -0
  7. data/Gemfile +8 -0
  8. data/Gemfile.lock +216 -0
  9. data/MIT-LICENCE +18 -0
  10. data/README.md +831 -0
  11. data/Rakefile +17 -0
  12. data/bin/karafka +7 -0
  13. data/karafka.gemspec +34 -0
  14. data/lib/karafka.rb +73 -0
  15. data/lib/karafka/app.rb +45 -0
  16. data/lib/karafka/base_controller.rb +162 -0
  17. data/lib/karafka/base_responder.rb +118 -0
  18. data/lib/karafka/base_worker.rb +41 -0
  19. data/lib/karafka/capistrano.rb +2 -0
  20. data/lib/karafka/capistrano/karafka.cap +84 -0
  21. data/lib/karafka/cli.rb +52 -0
  22. data/lib/karafka/cli/base.rb +74 -0
  23. data/lib/karafka/cli/console.rb +23 -0
  24. data/lib/karafka/cli/flow.rb +46 -0
  25. data/lib/karafka/cli/info.rb +26 -0
  26. data/lib/karafka/cli/install.rb +45 -0
  27. data/lib/karafka/cli/routes.rb +39 -0
  28. data/lib/karafka/cli/server.rb +59 -0
  29. data/lib/karafka/cli/worker.rb +26 -0
  30. data/lib/karafka/connection/consumer.rb +29 -0
  31. data/lib/karafka/connection/listener.rb +54 -0
  32. data/lib/karafka/connection/message.rb +17 -0
  33. data/lib/karafka/connection/topic_consumer.rb +48 -0
  34. data/lib/karafka/errors.rb +50 -0
  35. data/lib/karafka/fetcher.rb +40 -0
  36. data/lib/karafka/helpers/class_matcher.rb +77 -0
  37. data/lib/karafka/helpers/multi_delegator.rb +31 -0
  38. data/lib/karafka/loader.rb +77 -0
  39. data/lib/karafka/logger.rb +52 -0
  40. data/lib/karafka/monitor.rb +82 -0
  41. data/lib/karafka/params/interchanger.rb +33 -0
  42. data/lib/karafka/params/params.rb +102 -0
  43. data/lib/karafka/patches/dry/configurable/config.rb +37 -0
  44. data/lib/karafka/process.rb +61 -0
  45. data/lib/karafka/responders/builder.rb +33 -0
  46. data/lib/karafka/responders/topic.rb +43 -0
  47. data/lib/karafka/responders/usage_validator.rb +59 -0
  48. data/lib/karafka/routing/builder.rb +89 -0
  49. data/lib/karafka/routing/route.rb +80 -0
  50. data/lib/karafka/routing/router.rb +38 -0
  51. data/lib/karafka/server.rb +53 -0
  52. data/lib/karafka/setup/config.rb +57 -0
  53. data/lib/karafka/setup/configurators/base.rb +33 -0
  54. data/lib/karafka/setup/configurators/celluloid.rb +20 -0
  55. data/lib/karafka/setup/configurators/sidekiq.rb +34 -0
  56. data/lib/karafka/setup/configurators/water_drop.rb +19 -0
  57. data/lib/karafka/setup/configurators/worker_glass.rb +13 -0
  58. data/lib/karafka/status.rb +23 -0
  59. data/lib/karafka/templates/app.rb.example +26 -0
  60. data/lib/karafka/templates/application_controller.rb.example +5 -0
  61. data/lib/karafka/templates/application_responder.rb.example +9 -0
  62. data/lib/karafka/templates/application_worker.rb.example +12 -0
  63. data/lib/karafka/templates/config.ru.example +13 -0
  64. data/lib/karafka/templates/sidekiq.yml.example +26 -0
  65. data/lib/karafka/version.rb +6 -0
  66. data/lib/karafka/workers/builder.rb +49 -0
  67. data/log/.gitkeep +0 -0
  68. metadata +267 -0
@@ -0,0 +1,19 @@
1
+ module Karafka
2
+ module Setup
3
+ class Configurators
4
+ # Class responsible for setting up WaterDrop configuration
5
+ class WaterDrop < Base
6
+ # Sets up a WaterDrop settings
7
+ def setup
8
+ ::WaterDrop.setup do |water_config|
9
+ water_config.send_messages = true
10
+ water_config.connection_pool_size = config.concurrency
11
+ water_config.connection_pool_timeout = 1
12
+ water_config.kafka_hosts = config.kafka.hosts
13
+ water_config.raise_on_failure = true
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,13 @@
1
+ module Karafka
2
+ module Setup
3
+ class Configurators
4
+ # Class responsible for setting up WorkerGlass settings
5
+ class WorkerGlass < Base
6
+ # Sets up a Karafka logger as celluloid logger
7
+ def setup
8
+ ::WorkerGlass.logger = ::Karafka.logger
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,23 @@
1
+ module Karafka
2
+ # App status monitor
3
+ class Status
4
+ include Singleton
5
+
6
+ # Available states and their transitions
7
+ STATES = {
8
+ initializing: :initialize!,
9
+ running: :run!,
10
+ stopped: :stop!
11
+ }.freeze
12
+
13
+ STATES.each do |state, transition|
14
+ define_method :"#{state}?" do
15
+ @status == state
16
+ end
17
+
18
+ define_method transition do
19
+ @status = state
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,26 @@
1
+ ENV['RACK_ENV'] ||= 'development'
2
+ ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
3
+
4
+ Bundler.require(:default, ENV['KARAFKA_ENV'])
5
+
6
+ Karafka::Loader.new.load(Karafka::App.root)
7
+
8
+ # App class
9
+ class App < Karafka::App
10
+ setup do |config|
11
+ config.kafka = { hosts: %w( 127.0.0.1:9092 ) }
12
+ config.name = 'example_app'
13
+ config.redis = {
14
+ url: 'redis://localhost:6379'
15
+ }
16
+ end
17
+
18
+ routes.draw do
19
+ # topic :example do
20
+ # controller ExampleController
21
+ # interchanger CustomInterchanger
22
+ # end
23
+ end
24
+ end
25
+
26
+ App.boot!
@@ -0,0 +1,5 @@
1
+ # Application controller from which all Karafka controllers should inherit
2
+ # You can rename it if it would conflict with your current code base (in case you're integrating
3
+ # Karafka with other frameworks)
4
+ class ApplicationController < Karafka::BaseController
5
+ end
@@ -0,0 +1,9 @@
1
+ # Application responder from which all Karafka responders should inherit
2
+ # You can rename it if it would conflict with your current code base (in case you're integrating
3
+ # Karafka with other frameworks)
4
+ class ApplicationResponder < Karafka::BaseResponder
5
+ # This method needs to be implemented in each of the responders
6
+ # def respond(data)
7
+ # respond_to :topic, data.to_json
8
+ # end
9
+ end
@@ -0,0 +1,12 @@
1
+ # Application worker from which all workers should inherit
2
+ # You can rename it if it would conflict with your current code base (in case you're integrating
3
+ # Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker
4
+ # to build worker classes
5
+ class ApplicationWorker < Karafka::BaseWorker
6
+ # You can disable WorkerGlass components if you want to
7
+ prepend WorkerGlass::Timeout
8
+ prepend WorkerGlass::Reentrancy
9
+
10
+ # If you remove WorkerGlass::Timeout, this line will be useless as well
11
+ self.timeout = 60
12
+ end
@@ -0,0 +1,13 @@
1
+ %w(
2
+ puma
3
+ sidekiq/web
4
+ ).each { |lib| require lib }
5
+
6
+ require Karafka.boot_file
7
+
8
+ use Rack::Auth::Basic, 'Protected Area' do |username, password|
9
+ username == 'sidekiq' &&
10
+ password == 'Pa$$WorD!'
11
+ end
12
+
13
+ run Sidekiq::Web
@@ -0,0 +1,26 @@
1
+ # Sample configuration file for Sidekiq.
2
+ # Options here can still be overridden by cmd line args.
3
+ # Place this file at config/sidekiq.yml and Sidekiq will
4
+ # pick it up automatically.
5
+
6
+ :verbose: true
7
+ :concurrency: 10
8
+
9
+ # Set timeout to 8 on Heroku, longer if you manage your own systems.
10
+ :timeout: 8
11
+
12
+ # Sidekiq will run this file through ERB when reading it so you can
13
+ # even put in dynamic logic, like a host-specific queue.
14
+ ## http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
15
+ #:queues:
16
+ # - all
17
+ # - high
18
+ # - highest321
19
+
20
+ # you can override concurrency based on environment
21
+ production:
22
+ :concurrency: 10
23
+ development:
24
+ :concurrency: 1
25
+ test:
26
+ :concurrency: 1
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+ # Main module namespace
3
+ module Karafka
4
+ # Current Karafka version
5
+ VERSION = '0.5.0'
6
+ end
@@ -0,0 +1,49 @@
1
+ module Karafka
2
+ # Internal stuff related to workers
3
+ module Workers
4
+ # Builder is used to check if there is a proper controller with the same name as
5
+ # a controller and if not, it will create a default one using Karafka::BaseWorker
6
+ # This is used as a building layer between controllers and workers. it will be only used
7
+ # when user does not provide his own worker that should perform controller stuff
8
+ class Builder
9
+ # @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
10
+ # @example Create a worker builder
11
+ # Karafka::Workers::Builder.new(SuperController)
12
+ def initialize(controller_class)
13
+ @controller_class = controller_class
14
+ end
15
+
16
+ # @return [Class] Sidekiq worker class that already exists or new build based
17
+ # on the provided controller_class name
18
+ # @example Controller: SuperController
19
+ # build #=> SuperWorker
20
+ # @example Controller: Videos::NewVideosController
21
+ # build #=> Videos::NewVideosWorker
22
+ def build
23
+ return matcher.match if matcher.match
24
+ klass = Class.new(base)
25
+ matcher.scope.const_set(matcher.name, klass)
26
+ end
27
+
28
+ private
29
+
30
+ # @return [Class] descendant of Karafka::BaseWorker from which all other workers
31
+ # should inherit
32
+ # @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
33
+ # direct Karafka::BaseWorker descendant from which it could build workers
34
+ def base
35
+ Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
36
+ end
37
+
38
+ # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
39
+ # and appropriate worker
40
+ def matcher
41
+ @matcher ||= Helpers::ClassMatcher.new(
42
+ @controller_class,
43
+ from: 'Controller',
44
+ to: 'Worker'
45
+ )
46
+ end
47
+ end
48
+ end
49
+ end
File without changes
metadata ADDED
@@ -0,0 +1,267 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: karafka
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.5.0
5
+ platform: ruby
6
+ authors:
7
+ - Maciej Mensfeld
8
+ - Pavlo Vavruk
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2016-09-30 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - "~>"
19
+ - !ruby/object:Gem::Version
20
+ version: '1.2'
21
+ type: :development
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - "~>"
26
+ - !ruby/object:Gem::Version
27
+ version: '1.2'
28
+ - !ruby/object:Gem::Dependency
29
+ name: ruby-kafka
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - '='
33
+ - !ruby/object:Gem::Version
34
+ version: 0.3.15
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - '='
40
+ - !ruby/object:Gem::Version
41
+ version: 0.3.15
42
+ - !ruby/object:Gem::Dependency
43
+ name: sidekiq
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - "~>"
47
+ - !ruby/object:Gem::Version
48
+ version: '4.2'
49
+ type: :runtime
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - "~>"
54
+ - !ruby/object:Gem::Version
55
+ version: '4.2'
56
+ - !ruby/object:Gem::Dependency
57
+ name: worker-glass
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - "~>"
61
+ - !ruby/object:Gem::Version
62
+ version: '0.2'
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '0.2'
70
+ - !ruby/object:Gem::Dependency
71
+ name: celluloid
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - "~>"
75
+ - !ruby/object:Gem::Version
76
+ version: '0.17'
77
+ type: :runtime
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - "~>"
82
+ - !ruby/object:Gem::Version
83
+ version: '0.17'
84
+ - !ruby/object:Gem::Dependency
85
+ name: envlogic
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - "~>"
89
+ - !ruby/object:Gem::Version
90
+ version: '1.0'
91
+ type: :runtime
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - "~>"
96
+ - !ruby/object:Gem::Version
97
+ version: '1.0'
98
+ - !ruby/object:Gem::Dependency
99
+ name: waterdrop
100
+ requirement: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - "~>"
103
+ - !ruby/object:Gem::Version
104
+ version: '0.3'
105
+ type: :runtime
106
+ prerelease: false
107
+ version_requirements: !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - "~>"
110
+ - !ruby/object:Gem::Version
111
+ version: '0.3'
112
+ - !ruby/object:Gem::Dependency
113
+ name: rake
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - "~>"
117
+ - !ruby/object:Gem::Version
118
+ version: '11.3'
119
+ type: :runtime
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - "~>"
124
+ - !ruby/object:Gem::Version
125
+ version: '11.3'
126
+ - !ruby/object:Gem::Dependency
127
+ name: thor
128
+ requirement: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - "~>"
131
+ - !ruby/object:Gem::Version
132
+ version: '0.19'
133
+ type: :runtime
134
+ prerelease: false
135
+ version_requirements: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - "~>"
138
+ - !ruby/object:Gem::Version
139
+ version: '0.19'
140
+ - !ruby/object:Gem::Dependency
141
+ name: activesupport
142
+ requirement: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - "~>"
145
+ - !ruby/object:Gem::Version
146
+ version: '5.0'
147
+ type: :runtime
148
+ prerelease: false
149
+ version_requirements: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - "~>"
152
+ - !ruby/object:Gem::Version
153
+ version: '5.0'
154
+ - !ruby/object:Gem::Dependency
155
+ name: dry-configurable
156
+ requirement: !ruby/object:Gem::Requirement
157
+ requirements:
158
+ - - "~>"
159
+ - !ruby/object:Gem::Version
160
+ version: 0.1.7
161
+ type: :runtime
162
+ prerelease: false
163
+ version_requirements: !ruby/object:Gem::Requirement
164
+ requirements:
165
+ - - "~>"
166
+ - !ruby/object:Gem::Version
167
+ version: 0.1.7
168
+ description: " Microframework used to simplify Kafka based Ruby applications "
169
+ email:
170
+ - maciej@mensfeld.pl
171
+ - pavlo.vavruk@gmail.com
172
+ executables:
173
+ - karafka
174
+ extensions: []
175
+ extra_rdoc_files: []
176
+ files:
177
+ - ".gitignore"
178
+ - ".ruby-gemset"
179
+ - ".ruby-version"
180
+ - ".travis.yml"
181
+ - CHANGELOG.md
182
+ - Gemfile
183
+ - Gemfile.lock
184
+ - MIT-LICENCE
185
+ - README.md
186
+ - Rakefile
187
+ - bin/karafka
188
+ - karafka.gemspec
189
+ - lib/karafka.rb
190
+ - lib/karafka/app.rb
191
+ - lib/karafka/base_controller.rb
192
+ - lib/karafka/base_responder.rb
193
+ - lib/karafka/base_worker.rb
194
+ - lib/karafka/capistrano.rb
195
+ - lib/karafka/capistrano/karafka.cap
196
+ - lib/karafka/cli.rb
197
+ - lib/karafka/cli/base.rb
198
+ - lib/karafka/cli/console.rb
199
+ - lib/karafka/cli/flow.rb
200
+ - lib/karafka/cli/info.rb
201
+ - lib/karafka/cli/install.rb
202
+ - lib/karafka/cli/routes.rb
203
+ - lib/karafka/cli/server.rb
204
+ - lib/karafka/cli/worker.rb
205
+ - lib/karafka/connection/consumer.rb
206
+ - lib/karafka/connection/listener.rb
207
+ - lib/karafka/connection/message.rb
208
+ - lib/karafka/connection/topic_consumer.rb
209
+ - lib/karafka/errors.rb
210
+ - lib/karafka/fetcher.rb
211
+ - lib/karafka/helpers/class_matcher.rb
212
+ - lib/karafka/helpers/multi_delegator.rb
213
+ - lib/karafka/loader.rb
214
+ - lib/karafka/logger.rb
215
+ - lib/karafka/monitor.rb
216
+ - lib/karafka/params/interchanger.rb
217
+ - lib/karafka/params/params.rb
218
+ - lib/karafka/patches/dry/configurable/config.rb
219
+ - lib/karafka/process.rb
220
+ - lib/karafka/responders/builder.rb
221
+ - lib/karafka/responders/topic.rb
222
+ - lib/karafka/responders/usage_validator.rb
223
+ - lib/karafka/routing/builder.rb
224
+ - lib/karafka/routing/route.rb
225
+ - lib/karafka/routing/router.rb
226
+ - lib/karafka/server.rb
227
+ - lib/karafka/setup/config.rb
228
+ - lib/karafka/setup/configurators/base.rb
229
+ - lib/karafka/setup/configurators/celluloid.rb
230
+ - lib/karafka/setup/configurators/sidekiq.rb
231
+ - lib/karafka/setup/configurators/water_drop.rb
232
+ - lib/karafka/setup/configurators/worker_glass.rb
233
+ - lib/karafka/status.rb
234
+ - lib/karafka/templates/app.rb.example
235
+ - lib/karafka/templates/application_controller.rb.example
236
+ - lib/karafka/templates/application_responder.rb.example
237
+ - lib/karafka/templates/application_worker.rb.example
238
+ - lib/karafka/templates/config.ru.example
239
+ - lib/karafka/templates/sidekiq.yml.example
240
+ - lib/karafka/version.rb
241
+ - lib/karafka/workers/builder.rb
242
+ - log/.gitkeep
243
+ homepage: https://github.com/karafka/karafka
244
+ licenses:
245
+ - MIT
246
+ metadata: {}
247
+ post_install_message:
248
+ rdoc_options: []
249
+ require_paths:
250
+ - lib
251
+ required_ruby_version: !ruby/object:Gem::Requirement
252
+ requirements:
253
+ - - ">="
254
+ - !ruby/object:Gem::Version
255
+ version: 2.3.0
256
+ required_rubygems_version: !ruby/object:Gem::Requirement
257
+ requirements:
258
+ - - ">="
259
+ - !ruby/object:Gem::Version
260
+ version: '0'
261
+ requirements: []
262
+ rubyforge_project:
263
+ rubygems_version: 2.5.1
264
+ signing_key:
265
+ specification_version: 4
266
+ summary: Ruby based Microframework for handling Apache Kafka incoming messages
267
+ test_files: []