karafka 0.6.0.rc2 → 1.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -2
- data/Gemfile.lock +4 -18
- data/karafka.gemspec +0 -1
- data/lib/karafka.rb +2 -12
- data/lib/karafka/attributes_map.rb +2 -3
- data/lib/karafka/backends/inline.rb +17 -0
- data/lib/karafka/base_controller.rb +40 -96
- data/lib/karafka/base_responder.rb +19 -19
- data/lib/karafka/cli/info.rb +2 -3
- data/lib/karafka/cli/install.rb +0 -3
- data/lib/karafka/connection/messages_processor.rb +10 -6
- data/lib/karafka/controllers/includer.rb +51 -0
- data/lib/karafka/controllers/responders.rb +19 -0
- data/lib/karafka/controllers/single_params.rb +15 -0
- data/lib/karafka/errors.rb +1 -17
- data/lib/karafka/fetcher.rb +2 -2
- data/lib/karafka/helpers/class_matcher.rb +9 -10
- data/lib/karafka/params/params.rb +2 -2
- data/lib/karafka/params/params_batch.rb +2 -7
- data/lib/karafka/persistence.rb +18 -0
- data/lib/karafka/routing/builder.rb +1 -1
- data/lib/karafka/routing/router.rb +3 -11
- data/lib/karafka/routing/topic.rb +1 -13
- data/lib/karafka/schemas/config.rb +1 -12
- data/lib/karafka/schemas/consumer_group.rb +2 -2
- data/lib/karafka/setup/config.rb +14 -19
- data/lib/karafka/templates/karafka.rb.example +1 -5
- data/lib/karafka/version.rb +1 -1
- metadata +8 -24
- data/lib/karafka/base_worker.rb +0 -26
- data/lib/karafka/cli/worker.rb +0 -28
- data/lib/karafka/params/interchanger.rb +0 -35
- data/lib/karafka/setup/configurators/sidekiq.rb +0 -36
- data/lib/karafka/templates/application_worker.rb.example +0 -8
- data/lib/karafka/templates/sidekiq.yml.example +0 -26
- data/lib/karafka/workers/builder.rb +0 -51
@@ -17,17 +17,13 @@ class KarafkaApp < Karafka::App
|
|
17
17
|
setup do |config|
|
18
18
|
config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
|
19
19
|
config.client_id = 'example_app'
|
20
|
-
config.
|
21
|
-
url: 'redis://localhost:6379'
|
22
|
-
}
|
23
|
-
config.processing_backend = Karafka.env.development? ? :inline : :sidekiq
|
20
|
+
config.backend = :inline
|
24
21
|
config.batch_consuming = true
|
25
22
|
end
|
26
23
|
|
27
24
|
consumer_groups.draw do
|
28
25
|
# topic :example do
|
29
26
|
# controller ExampleController
|
30
|
-
# interchanger CustomInterchanger
|
31
27
|
# end
|
32
28
|
|
33
29
|
# consumer_group :bigger_group do
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-
|
13
|
+
date: 2017-09-01 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: ruby-kafka
|
@@ -26,20 +26,6 @@ dependencies:
|
|
26
26
|
- - ">="
|
27
27
|
- !ruby/object:Gem::Version
|
28
28
|
version: '0.4'
|
29
|
-
- !ruby/object:Gem::Dependency
|
30
|
-
name: sidekiq
|
31
|
-
requirement: !ruby/object:Gem::Requirement
|
32
|
-
requirements:
|
33
|
-
- - ">="
|
34
|
-
- !ruby/object:Gem::Version
|
35
|
-
version: '4.2'
|
36
|
-
type: :runtime
|
37
|
-
prerelease: false
|
38
|
-
version_requirements: !ruby/object:Gem::Requirement
|
39
|
-
requirements:
|
40
|
-
- - ">="
|
41
|
-
- !ruby/object:Gem::Version
|
42
|
-
version: '4.2'
|
43
29
|
- !ruby/object:Gem::Dependency
|
44
30
|
name: celluloid
|
45
31
|
requirement: !ruby/object:Gem::Requirement
|
@@ -210,9 +196,9 @@ files:
|
|
210
196
|
- lib/karafka.rb
|
211
197
|
- lib/karafka/app.rb
|
212
198
|
- lib/karafka/attributes_map.rb
|
199
|
+
- lib/karafka/backends/inline.rb
|
213
200
|
- lib/karafka/base_controller.rb
|
214
201
|
- lib/karafka/base_responder.rb
|
215
|
-
- lib/karafka/base_worker.rb
|
216
202
|
- lib/karafka/cli.rb
|
217
203
|
- lib/karafka/cli/base.rb
|
218
204
|
- lib/karafka/cli/console.rb
|
@@ -220,11 +206,13 @@ files:
|
|
220
206
|
- lib/karafka/cli/info.rb
|
221
207
|
- lib/karafka/cli/install.rb
|
222
208
|
- lib/karafka/cli/server.rb
|
223
|
-
- lib/karafka/cli/worker.rb
|
224
209
|
- lib/karafka/connection/config_adapter.rb
|
225
210
|
- lib/karafka/connection/listener.rb
|
226
211
|
- lib/karafka/connection/messages_consumer.rb
|
227
212
|
- lib/karafka/connection/messages_processor.rb
|
213
|
+
- lib/karafka/controllers/includer.rb
|
214
|
+
- lib/karafka/controllers/responders.rb
|
215
|
+
- lib/karafka/controllers/single_params.rb
|
228
216
|
- lib/karafka/errors.rb
|
229
217
|
- lib/karafka/fetcher.rb
|
230
218
|
- lib/karafka/helpers/class_matcher.rb
|
@@ -233,11 +221,11 @@ files:
|
|
233
221
|
- lib/karafka/loader.rb
|
234
222
|
- lib/karafka/logger.rb
|
235
223
|
- lib/karafka/monitor.rb
|
236
|
-
- lib/karafka/params/interchanger.rb
|
237
224
|
- lib/karafka/params/params.rb
|
238
225
|
- lib/karafka/params/params_batch.rb
|
239
226
|
- lib/karafka/parsers/json.rb
|
240
227
|
- lib/karafka/patches/dry_configurable.rb
|
228
|
+
- lib/karafka/persistence.rb
|
241
229
|
- lib/karafka/process.rb
|
242
230
|
- lib/karafka/responders/builder.rb
|
243
231
|
- lib/karafka/responders/topic.rb
|
@@ -255,16 +243,12 @@ files:
|
|
255
243
|
- lib/karafka/setup/config.rb
|
256
244
|
- lib/karafka/setup/configurators/base.rb
|
257
245
|
- lib/karafka/setup/configurators/celluloid.rb
|
258
|
-
- lib/karafka/setup/configurators/sidekiq.rb
|
259
246
|
- lib/karafka/setup/configurators/water_drop.rb
|
260
247
|
- lib/karafka/status.rb
|
261
248
|
- lib/karafka/templates/application_controller.rb.example
|
262
249
|
- lib/karafka/templates/application_responder.rb.example
|
263
|
-
- lib/karafka/templates/application_worker.rb.example
|
264
250
|
- lib/karafka/templates/karafka.rb.example
|
265
|
-
- lib/karafka/templates/sidekiq.yml.example
|
266
251
|
- lib/karafka/version.rb
|
267
|
-
- lib/karafka/workers/builder.rb
|
268
252
|
- log/.gitkeep
|
269
253
|
homepage: https://github.com/karafka/karafka
|
270
254
|
licenses:
|
@@ -286,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
286
270
|
version: 1.3.1
|
287
271
|
requirements: []
|
288
272
|
rubyforge_project:
|
289
|
-
rubygems_version: 2.6.
|
273
|
+
rubygems_version: 2.6.13
|
290
274
|
signing_key:
|
291
275
|
specification_version: 4
|
292
276
|
summary: Ruby based framework for working with Apache Kafka
|
data/lib/karafka/base_worker.rb
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# Worker wrapper for Sidekiq workers
|
5
|
-
class BaseWorker
|
6
|
-
include Sidekiq::Worker
|
7
|
-
|
8
|
-
# Executes the logic that lies in #perform Karafka controller method
|
9
|
-
# @param topic_id [String] Unique topic id that we will use to find a proper topic
|
10
|
-
# @param params_batch [Array] Array with messages batch
|
11
|
-
def perform(topic_id, params_batch)
|
12
|
-
Karafka.monitor.notice(self.class, params_batch)
|
13
|
-
controller(topic_id, params_batch).call
|
14
|
-
end
|
15
|
-
|
16
|
-
private
|
17
|
-
|
18
|
-
# @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
|
19
|
-
# with params_batch assigned already (controller is ready to use)
|
20
|
-
def controller(topic_id, params_batch)
|
21
|
-
@controller ||= Karafka::Routing::Router.build(topic_id).tap do |ctrl|
|
22
|
-
ctrl.params_batch = ctrl.topic.interchanger.parse(params_batch)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
data/lib/karafka/cli/worker.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# Karafka framework Cli
|
5
|
-
class Cli < Thor
|
6
|
-
# Worker Karafka Cli action
|
7
|
-
class Worker < Base
|
8
|
-
desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
|
9
|
-
option aliases: 'w'
|
10
|
-
|
11
|
-
# Start the Karafka Sidekiq worker
|
12
|
-
# @param params [Array<String>] additional params that will be passed to sidekiq, that way we
|
13
|
-
# can override any default Karafka settings
|
14
|
-
def call(*params)
|
15
|
-
puts 'Starting Karafka worker'
|
16
|
-
config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
|
17
|
-
req = "-r #{Karafka.boot_file}"
|
18
|
-
env = "-e #{Karafka.env}"
|
19
|
-
|
20
|
-
cli.info
|
21
|
-
|
22
|
-
cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
|
23
|
-
puts(cmd)
|
24
|
-
exec(cmd)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Params
|
5
|
-
# Interchangers allow us to format/encode/pack data that is being send to perform_async
|
6
|
-
# This is meant to target mostly issues with data encoding like this one:
|
7
|
-
# https://github.com/mperham/sidekiq/issues/197
|
8
|
-
# Each custom interchanger should implement following methods:
|
9
|
-
# - load - it is meant to encode params before they get stored inside Redis
|
10
|
-
# - parse - decoded params back to a hash format that we can use
|
11
|
-
class Interchanger
|
12
|
-
class << self
|
13
|
-
# @param params [Karafka::Params::Params] Karafka params object
|
14
|
-
# @note Params might not be parsed because of lazy loading feature. If you implement your
|
15
|
-
# own interchanger logic, this method needs to return data that can be converted to
|
16
|
-
# json with default Sidekiqs logic
|
17
|
-
# @return [Karafka::Params::Params] same as input. We assume that our incoming data is
|
18
|
-
# jsonable-safe and we can rely on a direct Sidekiq encoding logic
|
19
|
-
def load(params)
|
20
|
-
params
|
21
|
-
end
|
22
|
-
|
23
|
-
# @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
|
24
|
-
# @note Hash is what we need to build Karafka::Params::Params, so we do nothing
|
25
|
-
# with it. If you implement your own interchanger logic, this method needs to return
|
26
|
-
# a hash with appropriate data that will be used to build Karafka::Params::Params
|
27
|
-
# @return [Hash] We return exactly what we received. We rely on sidekiqs default
|
28
|
-
# interchanging format
|
29
|
-
def parse(params)
|
30
|
-
params
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Setup
|
5
|
-
class Configurators
|
6
|
-
# Class to configure all the Sidekiq settings based on Karafka settings
|
7
|
-
class Sidekiq < Base
|
8
|
-
# Sets up sidekiq client and server
|
9
|
-
def setup
|
10
|
-
setup_sidekiq_client
|
11
|
-
setup_sidekiq_server
|
12
|
-
end
|
13
|
-
|
14
|
-
private
|
15
|
-
|
16
|
-
# Configure sidekiq client
|
17
|
-
def setup_sidekiq_client
|
18
|
-
::Sidekiq.configure_client do |sidekiq_config|
|
19
|
-
sidekiq_config.redis = config.redis.to_h.merge(
|
20
|
-
size: config.concurrency
|
21
|
-
)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
# Configure sidekiq setorrver
|
26
|
-
def setup_sidekiq_server
|
27
|
-
::Sidekiq.configure_server do |sidekiq_config|
|
28
|
-
# We don't set size for the server - this will be set automatically based
|
29
|
-
# on the Sidekiq concurrency level (Sidekiq not Karafkas)
|
30
|
-
sidekiq_config.redis = config.redis.to_h
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,8 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Application worker from which all workers should inherit
|
4
|
-
# You can rename it if it would conflict with your current code base (in case you're integrating
|
5
|
-
# Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker
|
6
|
-
# to build worker classes
|
7
|
-
class ApplicationWorker < Karafka::BaseWorker
|
8
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
# Sample configuration file for Sidekiq.
|
2
|
-
# Options here can still be overridden by cmd line args.
|
3
|
-
# Place this file at config/sidekiq.yml and Sidekiq will
|
4
|
-
# pick it up automatically.
|
5
|
-
|
6
|
-
:verbose: true
|
7
|
-
:concurrency: 10
|
8
|
-
|
9
|
-
# Set timeout to 8 on Heroku, longer if you manage your own systems.
|
10
|
-
:timeout: 8
|
11
|
-
|
12
|
-
# Sidekiq will run this file through ERB when reading it so you can
|
13
|
-
# even put in dynamic logic, like a host-specific queue.
|
14
|
-
## http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
|
15
|
-
#:queues:
|
16
|
-
# - all
|
17
|
-
# - high
|
18
|
-
# - highest321
|
19
|
-
|
20
|
-
# you can override concurrency based on environment
|
21
|
-
production:
|
22
|
-
:concurrency: 10
|
23
|
-
development:
|
24
|
-
:concurrency: 1
|
25
|
-
test:
|
26
|
-
:concurrency: 1
|
@@ -1,51 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# Internal stuff related to workers
|
5
|
-
module Workers
|
6
|
-
# Builder is used to check if there is a proper controller with the same name as
|
7
|
-
# a controller and if not, it will create a default one using Karafka::BaseWorker
|
8
|
-
# This is used as a building layer between controllers and workers. it will be only used
|
9
|
-
# when user does not provide his own worker that should perform controller stuff
|
10
|
-
class Builder
|
11
|
-
# @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
|
12
|
-
# @example Create a worker builder
|
13
|
-
# Karafka::Workers::Builder.new(SuperController)
|
14
|
-
def initialize(controller_class)
|
15
|
-
@controller_class = controller_class
|
16
|
-
end
|
17
|
-
|
18
|
-
# @return [Class] Sidekiq worker class that already exists or new build based
|
19
|
-
# on the provided controller_class name
|
20
|
-
# @example Controller: SuperController
|
21
|
-
# build #=> SuperWorker
|
22
|
-
# @example Controller: Videos::NewVideosController
|
23
|
-
# build #=> Videos::NewVideosWorker
|
24
|
-
def build
|
25
|
-
return matcher.match if matcher.match
|
26
|
-
klass = Class.new(base)
|
27
|
-
matcher.scope.const_set(matcher.name, klass)
|
28
|
-
end
|
29
|
-
|
30
|
-
private
|
31
|
-
|
32
|
-
# @return [Class] descendant of Karafka::BaseWorker from which all other workers
|
33
|
-
# should inherit
|
34
|
-
# @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
|
35
|
-
# direct Karafka::BaseWorker descendant from which it could build workers
|
36
|
-
def base
|
37
|
-
Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
|
38
|
-
end
|
39
|
-
|
40
|
-
# @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
|
41
|
-
# and appropriate worker
|
42
|
-
def matcher
|
43
|
-
@matcher ||= Helpers::ClassMatcher.new(
|
44
|
-
@controller_class,
|
45
|
-
from: 'Controller',
|
46
|
-
to: 'Worker'
|
47
|
-
)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|