karafka 0.6.0.rc2 → 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -2
  3. data/Gemfile.lock +4 -18
  4. data/karafka.gemspec +0 -1
  5. data/lib/karafka.rb +2 -12
  6. data/lib/karafka/attributes_map.rb +2 -3
  7. data/lib/karafka/backends/inline.rb +17 -0
  8. data/lib/karafka/base_controller.rb +40 -96
  9. data/lib/karafka/base_responder.rb +19 -19
  10. data/lib/karafka/cli/info.rb +2 -3
  11. data/lib/karafka/cli/install.rb +0 -3
  12. data/lib/karafka/connection/messages_processor.rb +10 -6
  13. data/lib/karafka/controllers/includer.rb +51 -0
  14. data/lib/karafka/controllers/responders.rb +19 -0
  15. data/lib/karafka/controllers/single_params.rb +15 -0
  16. data/lib/karafka/errors.rb +1 -17
  17. data/lib/karafka/fetcher.rb +2 -2
  18. data/lib/karafka/helpers/class_matcher.rb +9 -10
  19. data/lib/karafka/params/params.rb +2 -2
  20. data/lib/karafka/params/params_batch.rb +2 -7
  21. data/lib/karafka/persistence.rb +18 -0
  22. data/lib/karafka/routing/builder.rb +1 -1
  23. data/lib/karafka/routing/router.rb +3 -11
  24. data/lib/karafka/routing/topic.rb +1 -13
  25. data/lib/karafka/schemas/config.rb +1 -12
  26. data/lib/karafka/schemas/consumer_group.rb +2 -2
  27. data/lib/karafka/setup/config.rb +14 -19
  28. data/lib/karafka/templates/karafka.rb.example +1 -5
  29. data/lib/karafka/version.rb +1 -1
  30. metadata +8 -24
  31. data/lib/karafka/base_worker.rb +0 -26
  32. data/lib/karafka/cli/worker.rb +0 -28
  33. data/lib/karafka/params/interchanger.rb +0 -35
  34. data/lib/karafka/setup/configurators/sidekiq.rb +0 -36
  35. data/lib/karafka/templates/application_worker.rb.example +0 -8
  36. data/lib/karafka/templates/sidekiq.yml.example +0 -26
  37. data/lib/karafka/workers/builder.rb +0 -51
@@ -17,17 +17,13 @@ class KarafkaApp < Karafka::App
17
17
  setup do |config|
18
18
  config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
19
19
  config.client_id = 'example_app'
20
- config.redis = {
21
- url: 'redis://localhost:6379'
22
- }
23
- config.processing_backend = Karafka.env.development? ? :inline : :sidekiq
20
+ config.backend = :inline
24
21
  config.batch_consuming = true
25
22
  end
26
23
 
27
24
  consumer_groups.draw do
28
25
  # topic :example do
29
26
  # controller ExampleController
30
- # interchanger CustomInterchanger
31
27
  # end
32
28
 
33
29
  # consumer_group :bigger_group do
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '0.6.0.rc2'
6
+ VERSION = '1.0.0.rc1'
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0.rc2
4
+ version: 1.0.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-08-23 00:00:00.000000000 Z
13
+ date: 2017-09-01 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: ruby-kafka
@@ -26,20 +26,6 @@ dependencies:
26
26
  - - ">="
27
27
  - !ruby/object:Gem::Version
28
28
  version: '0.4'
29
- - !ruby/object:Gem::Dependency
30
- name: sidekiq
31
- requirement: !ruby/object:Gem::Requirement
32
- requirements:
33
- - - ">="
34
- - !ruby/object:Gem::Version
35
- version: '4.2'
36
- type: :runtime
37
- prerelease: false
38
- version_requirements: !ruby/object:Gem::Requirement
39
- requirements:
40
- - - ">="
41
- - !ruby/object:Gem::Version
42
- version: '4.2'
43
29
  - !ruby/object:Gem::Dependency
44
30
  name: celluloid
45
31
  requirement: !ruby/object:Gem::Requirement
@@ -210,9 +196,9 @@ files:
210
196
  - lib/karafka.rb
211
197
  - lib/karafka/app.rb
212
198
  - lib/karafka/attributes_map.rb
199
+ - lib/karafka/backends/inline.rb
213
200
  - lib/karafka/base_controller.rb
214
201
  - lib/karafka/base_responder.rb
215
- - lib/karafka/base_worker.rb
216
202
  - lib/karafka/cli.rb
217
203
  - lib/karafka/cli/base.rb
218
204
  - lib/karafka/cli/console.rb
@@ -220,11 +206,13 @@ files:
220
206
  - lib/karafka/cli/info.rb
221
207
  - lib/karafka/cli/install.rb
222
208
  - lib/karafka/cli/server.rb
223
- - lib/karafka/cli/worker.rb
224
209
  - lib/karafka/connection/config_adapter.rb
225
210
  - lib/karafka/connection/listener.rb
226
211
  - lib/karafka/connection/messages_consumer.rb
227
212
  - lib/karafka/connection/messages_processor.rb
213
+ - lib/karafka/controllers/includer.rb
214
+ - lib/karafka/controllers/responders.rb
215
+ - lib/karafka/controllers/single_params.rb
228
216
  - lib/karafka/errors.rb
229
217
  - lib/karafka/fetcher.rb
230
218
  - lib/karafka/helpers/class_matcher.rb
@@ -233,11 +221,11 @@ files:
233
221
  - lib/karafka/loader.rb
234
222
  - lib/karafka/logger.rb
235
223
  - lib/karafka/monitor.rb
236
- - lib/karafka/params/interchanger.rb
237
224
  - lib/karafka/params/params.rb
238
225
  - lib/karafka/params/params_batch.rb
239
226
  - lib/karafka/parsers/json.rb
240
227
  - lib/karafka/patches/dry_configurable.rb
228
+ - lib/karafka/persistence.rb
241
229
  - lib/karafka/process.rb
242
230
  - lib/karafka/responders/builder.rb
243
231
  - lib/karafka/responders/topic.rb
@@ -255,16 +243,12 @@ files:
255
243
  - lib/karafka/setup/config.rb
256
244
  - lib/karafka/setup/configurators/base.rb
257
245
  - lib/karafka/setup/configurators/celluloid.rb
258
- - lib/karafka/setup/configurators/sidekiq.rb
259
246
  - lib/karafka/setup/configurators/water_drop.rb
260
247
  - lib/karafka/status.rb
261
248
  - lib/karafka/templates/application_controller.rb.example
262
249
  - lib/karafka/templates/application_responder.rb.example
263
- - lib/karafka/templates/application_worker.rb.example
264
250
  - lib/karafka/templates/karafka.rb.example
265
- - lib/karafka/templates/sidekiq.yml.example
266
251
  - lib/karafka/version.rb
267
- - lib/karafka/workers/builder.rb
268
252
  - log/.gitkeep
269
253
  homepage: https://github.com/karafka/karafka
270
254
  licenses:
@@ -286,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
286
270
  version: 1.3.1
287
271
  requirements: []
288
272
  rubyforge_project:
289
- rubygems_version: 2.6.11
273
+ rubygems_version: 2.6.13
290
274
  signing_key:
291
275
  specification_version: 4
292
276
  summary: Ruby based framework for working with Apache Kafka
@@ -1,26 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Worker wrapper for Sidekiq workers
5
- class BaseWorker
6
- include Sidekiq::Worker
7
-
8
- # Executes the logic that lies in #perform Karafka controller method
9
- # @param topic_id [String] Unique topic id that we will use to find a proper topic
10
- # @param params_batch [Array] Array with messages batch
11
- def perform(topic_id, params_batch)
12
- Karafka.monitor.notice(self.class, params_batch)
13
- controller(topic_id, params_batch).call
14
- end
15
-
16
- private
17
-
18
- # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
19
- # with params_batch assigned already (controller is ready to use)
20
- def controller(topic_id, params_batch)
21
- @controller ||= Karafka::Routing::Router.build(topic_id).tap do |ctrl|
22
- ctrl.params_batch = ctrl.topic.interchanger.parse(params_batch)
23
- end
24
- end
25
- end
26
- end
@@ -1,28 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Karafka framework Cli
5
- class Cli < Thor
6
- # Worker Karafka Cli action
7
- class Worker < Base
8
- desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
9
- option aliases: 'w'
10
-
11
- # Start the Karafka Sidekiq worker
12
- # @param params [Array<String>] additional params that will be passed to sidekiq, that way we
13
- # can override any default Karafka settings
14
- def call(*params)
15
- puts 'Starting Karafka worker'
16
- config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
17
- req = "-r #{Karafka.boot_file}"
18
- env = "-e #{Karafka.env}"
19
-
20
- cli.info
21
-
22
- cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
23
- puts(cmd)
24
- exec(cmd)
25
- end
26
- end
27
- end
28
- end
@@ -1,35 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Params
5
- # Interchangers allow us to format/encode/pack data that is being send to perform_async
6
- # This is meant to target mostly issues with data encoding like this one:
7
- # https://github.com/mperham/sidekiq/issues/197
8
- # Each custom interchanger should implement following methods:
9
- # - load - it is meant to encode params before they get stored inside Redis
10
- # - parse - decoded params back to a hash format that we can use
11
- class Interchanger
12
- class << self
13
- # @param params [Karafka::Params::Params] Karafka params object
14
- # @note Params might not be parsed because of lazy loading feature. If you implement your
15
- # own interchanger logic, this method needs to return data that can be converted to
16
- # json with default Sidekiqs logic
17
- # @return [Karafka::Params::Params] same as input. We assume that our incoming data is
18
- # jsonable-safe and we can rely on a direct Sidekiq encoding logic
19
- def load(params)
20
- params
21
- end
22
-
23
- # @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
24
- # @note Hash is what we need to build Karafka::Params::Params, so we do nothing
25
- # with it. If you implement your own interchanger logic, this method needs to return
26
- # a hash with appropriate data that will be used to build Karafka::Params::Params
27
- # @return [Hash] We return exactly what we received. We rely on sidekiqs default
28
- # interchanging format
29
- def parse(params)
30
- params
31
- end
32
- end
33
- end
34
- end
35
- end
@@ -1,36 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Setup
5
- class Configurators
6
- # Class to configure all the Sidekiq settings based on Karafka settings
7
- class Sidekiq < Base
8
- # Sets up sidekiq client and server
9
- def setup
10
- setup_sidekiq_client
11
- setup_sidekiq_server
12
- end
13
-
14
- private
15
-
16
- # Configure sidekiq client
17
- def setup_sidekiq_client
18
- ::Sidekiq.configure_client do |sidekiq_config|
19
- sidekiq_config.redis = config.redis.to_h.merge(
20
- size: config.concurrency
21
- )
22
- end
23
- end
24
-
25
- # Configure sidekiq setorrver
26
- def setup_sidekiq_server
27
- ::Sidekiq.configure_server do |sidekiq_config|
28
- # We don't set size for the server - this will be set automatically based
29
- # on the Sidekiq concurrency level (Sidekiq not Karafkas)
30
- sidekiq_config.redis = config.redis.to_h
31
- end
32
- end
33
- end
34
- end
35
- end
36
- end
@@ -1,8 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Application worker from which all workers should inherit
4
- # You can rename it if it would conflict with your current code base (in case you're integrating
5
- # Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker
6
- # to build worker classes
7
- class ApplicationWorker < Karafka::BaseWorker
8
- end
@@ -1,26 +0,0 @@
1
- # Sample configuration file for Sidekiq.
2
- # Options here can still be overridden by cmd line args.
3
- # Place this file at config/sidekiq.yml and Sidekiq will
4
- # pick it up automatically.
5
-
6
- :verbose: true
7
- :concurrency: 10
8
-
9
- # Set timeout to 8 on Heroku, longer if you manage your own systems.
10
- :timeout: 8
11
-
12
- # Sidekiq will run this file through ERB when reading it so you can
13
- # even put in dynamic logic, like a host-specific queue.
14
- ## http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
15
- #:queues:
16
- # - all
17
- # - high
18
- # - highest321
19
-
20
- # you can override concurrency based on environment
21
- production:
22
- :concurrency: 10
23
- development:
24
- :concurrency: 1
25
- test:
26
- :concurrency: 1
@@ -1,51 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Internal stuff related to workers
5
- module Workers
6
- # Builder is used to check if there is a proper controller with the same name as
7
- # a controller and if not, it will create a default one using Karafka::BaseWorker
8
- # This is used as a building layer between controllers and workers. it will be only used
9
- # when user does not provide his own worker that should perform controller stuff
10
- class Builder
11
- # @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
12
- # @example Create a worker builder
13
- # Karafka::Workers::Builder.new(SuperController)
14
- def initialize(controller_class)
15
- @controller_class = controller_class
16
- end
17
-
18
- # @return [Class] Sidekiq worker class that already exists or new build based
19
- # on the provided controller_class name
20
- # @example Controller: SuperController
21
- # build #=> SuperWorker
22
- # @example Controller: Videos::NewVideosController
23
- # build #=> Videos::NewVideosWorker
24
- def build
25
- return matcher.match if matcher.match
26
- klass = Class.new(base)
27
- matcher.scope.const_set(matcher.name, klass)
28
- end
29
-
30
- private
31
-
32
- # @return [Class] descendant of Karafka::BaseWorker from which all other workers
33
- # should inherit
34
- # @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
35
- # direct Karafka::BaseWorker descendant from which it could build workers
36
- def base
37
- Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
38
- end
39
-
40
- # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
41
- # and appropriate worker
42
- def matcher
43
- @matcher ||= Helpers::ClassMatcher.new(
44
- @controller_class,
45
- from: 'Controller',
46
- to: 'Worker'
47
- )
48
- end
49
- end
50
- end
51
- end