karafka 0.6.0.rc1 → 0.6.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +8 -2
- data/CHANGELOG.md +13 -2
- data/CONTRIBUTING.md +0 -25
- data/Gemfile +2 -0
- data/Gemfile.lock +8 -9
- data/README.md +22 -11
- data/Rakefile +1 -2
- data/bin/karafka +1 -1
- data/karafka.gemspec +2 -6
- data/lib/karafka.rb +9 -8
- data/lib/karafka/attributes_map.rb +3 -2
- data/lib/karafka/base_controller.rb +8 -1
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/console.rb +2 -2
- data/lib/karafka/cli/flow.rb +1 -1
- data/lib/karafka/cli/info.rb +2 -2
- data/lib/karafka/cli/install.rb +2 -2
- data/lib/karafka/cli/server.rb +14 -6
- data/lib/karafka/cli/worker.rb +1 -1
- data/lib/karafka/connection/config_adapter.rb +23 -22
- data/lib/karafka/connection/messages_consumer.rb +23 -3
- data/lib/karafka/connection/messages_processor.rb +6 -25
- data/lib/karafka/errors.rb +4 -0
- data/lib/karafka/fetcher.rb +1 -1
- data/lib/karafka/loader.rb +5 -55
- data/lib/karafka/monitor.rb +1 -1
- data/lib/karafka/params/params.rb +5 -1
- data/lib/karafka/parsers/json.rb +2 -2
- data/lib/karafka/routing/builder.rb +7 -0
- data/lib/karafka/routing/consumer_group.rb +10 -3
- data/lib/karafka/routing/topic.rb +1 -1
- data/lib/karafka/schemas/config.rb +4 -4
- data/lib/karafka/schemas/consumer_group.rb +2 -2
- data/lib/karafka/schemas/server_cli_options.rb +43 -0
- data/lib/karafka/server.rb +10 -0
- data/lib/karafka/setup/config.rb +9 -4
- data/lib/karafka/templates/application_controller.rb.example +2 -0
- data/lib/karafka/templates/application_responder.rb.example +2 -0
- data/lib/karafka/templates/application_worker.rb.example +2 -0
- data/lib/karafka/templates/{app.rb.example → karafka.rb.example} +13 -6
- data/lib/karafka/version.rb +1 -1
- metadata +11 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f1ac31c4f943edba6e25886bc34b2bfad35e69fb
|
4
|
+
data.tar.gz: 488973afae1eb933b27dec7dd2fdcbb21fe6edae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e0ce78c37d1309d7b6adc51b4c59545df6eed3ee1dca0441c751b309090bd4d62e9da100fbe8fb7086af1b4f1d600ead0f9f7b873bf3d45e8f95092c30cdd76
|
7
|
+
data.tar.gz: d2555a9f9ca030df2d31a49d2a272e4d13b681ad53590708c6ed960004690761a0e3cf9e88ee000a0870d48d60e8987a7c0bf619c10cbfc5dad057fb374115b3
|
data/.travis.yml
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
language: ruby
|
2
|
+
sudo: false
|
2
3
|
rvm:
|
3
4
|
- 2.3.0
|
4
5
|
- 2.3.1
|
5
6
|
- 2.3.2
|
6
7
|
- 2.3.3
|
8
|
+
- 2.3.4
|
7
9
|
- 2.4.0
|
8
10
|
- 2.4.1
|
9
|
-
|
10
|
-
|
11
|
+
- jruby-9.1.12.0
|
12
|
+
script: bundle exec rspec spec/
|
13
|
+
env:
|
14
|
+
global:
|
15
|
+
- JRUBY_OPTS='--debug'
|
16
|
+
install: bundle install --jobs=3 --retry=3
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
-
## 0.6.0.
|
3
|
+
## 0.6.0.rc2 - released
|
4
4
|
|
5
5
|
### Closed issues:
|
6
6
|
|
@@ -20,11 +20,20 @@
|
|
20
20
|
- #175 - Allow single consumer to subscribe to multiple topics
|
21
21
|
- #178 - Remove parsing failover when cannot unparse data
|
22
22
|
- #174 - Extended config validation
|
23
|
-
-
|
23
|
+
- ~~#180 - Switch from JSON parser to yajl-ruby~~
|
24
24
|
- #181 - When responder is defined and not used due to ```respond_with``` not being triggered in the perform, it won't raise an exception.
|
25
25
|
- #188 - Rename name in config to client id
|
26
26
|
- #186 - Support ruby-kafka ```ssl_ca_cert_file_path``` config
|
27
27
|
- #189 - karafka console does not preserve history on exit
|
28
|
+
- #191 - Karafka 0.6.0rc1 does not work with jruby / now it does :-)
|
29
|
+
- Switch to multi json so everyone can use their favourite JSON parser
|
30
|
+
- Added jruby support in general and in Travis
|
31
|
+
- #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
|
32
|
+
- #96 - Karafka server - possiblity to run it only for a certain topics
|
33
|
+
- ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
|
34
|
+
- (optional) pausing upon processing failures ```pause_timeout```
|
35
|
+
- Karafka console main process no longer intercepts irb errors
|
36
|
+
- Wiki updates
|
28
37
|
|
29
38
|
### New features and improvements
|
30
39
|
|
@@ -38,6 +47,7 @@
|
|
38
47
|
|
39
48
|
### Incompatibilities
|
40
49
|
|
50
|
+
- Default boot file is renamed from app.rb to karafka.rb
|
41
51
|
- Removed worker glass as dependency (now and independent gem)
|
42
52
|
- ```kafka.hosts``` option renamed to ```kafka.seed_brokers``` - you don't need to provide all the hosts to work with Kafka
|
43
53
|
- ```start_from_beginning``` moved into kafka scope (```kafka.start_from_beginning```)
|
@@ -47,6 +57,7 @@
|
|
47
57
|
- Renamed content to value to better resemble ruby-kafka internal messages naming convention
|
48
58
|
- When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
|
49
59
|
- Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
|
60
|
+
- Renamed ```inline_processing``` to ```processing_backend``` to reach 1.0 future compatibility
|
50
61
|
|
51
62
|
### Other changes
|
52
63
|
- PolishGeeksDevTools removed (in favour of Coditsu)
|
data/CONTRIBUTING.md
CHANGED
@@ -40,28 +40,3 @@ You can also reach us at hello@karafka.opencollective.com.
|
|
40
40
|
|
41
41
|
Thank you to all the people who have already contributed to karafka!
|
42
42
|
<a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
|
43
|
-
|
44
|
-
|
45
|
-
### Backers
|
46
|
-
|
47
|
-
Thank you to all our backers! [[Become a backer](https://opencollective.com/karafka#backer)]
|
48
|
-
|
49
|
-
<a href="https://opencollective.com/karafka#backers" target="_blank"><img src="https://opencollective.com/karafka/backers.svg?width=890"></a>
|
50
|
-
|
51
|
-
|
52
|
-
### Sponsors
|
53
|
-
|
54
|
-
Thank you to all our sponsors! (please ask your company to also support this open source project by [becoming a sponsor](https://opencollective.com/karafka#sponsor))
|
55
|
-
|
56
|
-
<a href="https://opencollective.com/karafka/sponsor/0/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/0/avatar.svg"></a>
|
57
|
-
<a href="https://opencollective.com/karafka/sponsor/1/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/1/avatar.svg"></a>
|
58
|
-
<a href="https://opencollective.com/karafka/sponsor/2/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/2/avatar.svg"></a>
|
59
|
-
<a href="https://opencollective.com/karafka/sponsor/3/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/3/avatar.svg"></a>
|
60
|
-
<a href="https://opencollective.com/karafka/sponsor/4/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/4/avatar.svg"></a>
|
61
|
-
<a href="https://opencollective.com/karafka/sponsor/5/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/5/avatar.svg"></a>
|
62
|
-
<a href="https://opencollective.com/karafka/sponsor/6/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/6/avatar.svg"></a>
|
63
|
-
<a href="https://opencollective.com/karafka/sponsor/7/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/7/avatar.svg"></a>
|
64
|
-
<a href="https://opencollective.com/karafka/sponsor/8/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/8/avatar.svg"></a>
|
65
|
-
<a href="https://opencollective.com/karafka/sponsor/9/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/9/avatar.svg"></a>
|
66
|
-
|
67
|
-
<!-- This `CONTRIBUTING.md` is based on @nayafia's template https://github.com/nayafia/contributing-template -->
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,18 +1,19 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (0.6.0.
|
4
|
+
karafka (0.6.0.rc2)
|
5
5
|
activesupport (>= 5.0)
|
6
6
|
celluloid
|
7
7
|
dry-configurable (~> 0.7)
|
8
8
|
dry-validation (~> 0.11)
|
9
9
|
envlogic (~> 1.0)
|
10
|
+
multi_json (>= 1.12)
|
10
11
|
rake (>= 11.3)
|
12
|
+
require_all (>= 1.4)
|
11
13
|
ruby-kafka (>= 0.4)
|
12
14
|
sidekiq (>= 4.2)
|
13
15
|
thor (~> 0.19)
|
14
16
|
waterdrop (>= 0.4)
|
15
|
-
yajl-ruby (>= 1.3.0)
|
16
17
|
|
17
18
|
GEM
|
18
19
|
remote: https://rubygems.org/
|
@@ -22,7 +23,6 @@ GEM
|
|
22
23
|
i18n (~> 0.7)
|
23
24
|
minitest (~> 5.1)
|
24
25
|
tzinfo (~> 1.1)
|
25
|
-
byebug (9.0.6)
|
26
26
|
celluloid (0.17.3)
|
27
27
|
celluloid-essentials
|
28
28
|
celluloid-extras
|
@@ -81,12 +81,14 @@ GEM
|
|
81
81
|
inflecto (0.0.2)
|
82
82
|
json (2.1.0)
|
83
83
|
minitest (5.10.3)
|
84
|
+
multi_json (1.12.1)
|
84
85
|
null-logger (0.1.4)
|
85
86
|
rack (2.0.3)
|
86
87
|
rack-protection (2.0.0)
|
87
88
|
rack
|
88
89
|
rake (12.0.0)
|
89
90
|
redis (3.3.3)
|
91
|
+
require_all (1.4.0)
|
90
92
|
rspec (3.6.0)
|
91
93
|
rspec-core (~> 3.6.0)
|
92
94
|
rspec-expectations (~> 3.6.0)
|
@@ -112,7 +114,7 @@ GEM
|
|
112
114
|
json (>= 1.8, < 3)
|
113
115
|
simplecov-html (~> 0.10.0)
|
114
116
|
simplecov-html (0.10.2)
|
115
|
-
thor (0.
|
117
|
+
thor (0.20.0)
|
116
118
|
thread_safe (0.3.6)
|
117
119
|
timecop (0.9.1)
|
118
120
|
timers (4.1.2)
|
@@ -126,17 +128,14 @@ GEM
|
|
126
128
|
null-logger
|
127
129
|
rake
|
128
130
|
ruby-kafka (~> 0.4)
|
129
|
-
yajl-ruby (1.3.0)
|
130
131
|
|
131
132
|
PLATFORMS
|
132
133
|
ruby
|
133
134
|
|
134
135
|
DEPENDENCIES
|
135
|
-
bundler (~> 1.2)
|
136
|
-
byebug
|
137
136
|
karafka!
|
138
|
-
rspec
|
139
|
-
simplecov
|
137
|
+
rspec
|
138
|
+
simplecov
|
140
139
|
timecop
|
141
140
|
|
142
141
|
BUNDLED WITH
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-

|
2
2
|
|
3
3
|
[](https://travis-ci.org/karafka/karafka)
|
4
4
|
[](#backers) [](#sponsors) [](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
|
@@ -19,9 +19,28 @@ Karafka based applications can be easily deployed to any type of infrastructure,
|
|
19
19
|
* Capistrano
|
20
20
|
* Docker
|
21
21
|
|
22
|
-
##
|
22
|
+
## Getting started
|
23
|
+
|
24
|
+
If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
|
25
|
+
|
26
|
+
```bash
|
27
|
+
git clone https://github.com/karafka/karafka-example-app ./example_app
|
28
|
+
```
|
29
|
+
|
30
|
+
then, just bundle install all the dependencies:
|
31
|
+
|
32
|
+
```bash
|
33
|
+
cd ./example_app
|
34
|
+
bundle install
|
35
|
+
```
|
36
|
+
|
37
|
+
and follow the instructions from the [example app Wiki](https://github.com/karafka/karafka-example-app/blob/master/README.md).
|
38
|
+
|
39
|
+
**Note**: you need to ensure, that you have Kafka up and running and you need to configure Kafka seed_brokers in the ```karafka.rb``` file.
|
40
|
+
|
41
|
+
If you need more details and know how on how to start Karafka with a clean installation, read the [Getting started page](https://github.com/karafka/karafka/wiki/Getting-started) section of our Wiki.
|
23
42
|
|
24
|
-
|
43
|
+
## Support
|
25
44
|
|
26
45
|
Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything. It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
|
27
46
|
|
@@ -39,14 +58,6 @@ If you are interested in our commercial services, please contact [Maciej Mensfel
|
|
39
58
|
|
40
59
|
Karafka framework and Karafka team are __not__ related to Kafka streaming service called CloudKarafka in any matter. We don't recommend nor discourage usage of their platform.
|
41
60
|
|
42
|
-
## Requirements
|
43
|
-
|
44
|
-
In order to use Karafka framework, you need to have:
|
45
|
-
|
46
|
-
- Zookeeper (required by Kafka)
|
47
|
-
- Kafka (at least 0.9.0)
|
48
|
-
- Ruby (at least 2.3.0)
|
49
|
-
|
50
61
|
## Note on Patches/Pull Requests
|
51
62
|
|
52
63
|
Fork the project.
|
data/Rakefile
CHANGED
data/bin/karafka
CHANGED
data/karafka.gemspec
CHANGED
@@ -26,12 +26,8 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.add_dependency 'activesupport', '>= 5.0'
|
27
27
|
spec.add_dependency 'dry-validation', '~> 0.11'
|
28
28
|
spec.add_dependency 'dry-configurable', '~> 0.7'
|
29
|
-
spec.add_dependency '
|
30
|
-
|
31
|
-
spec.add_development_dependency 'bundler', '~> 1.2'
|
32
|
-
spec.add_development_dependency 'rspec', '>= 3.6'
|
33
|
-
spec.add_development_dependency 'simplecov', '>= 0.14'
|
34
|
-
spec.add_development_dependency 'byebug'
|
29
|
+
spec.add_dependency 'multi_json', '>= 1.12'
|
30
|
+
spec.add_dependency 'require_all', '>= 1.4'
|
35
31
|
|
36
32
|
spec.required_ruby_version = '>= 2.3.0'
|
37
33
|
|
data/lib/karafka.rb
CHANGED
@@ -16,7 +16,8 @@
|
|
16
16
|
envlogic
|
17
17
|
thor
|
18
18
|
fileutils
|
19
|
-
|
19
|
+
multi_json
|
20
|
+
require_all
|
20
21
|
dry-configurable
|
21
22
|
dry-validation
|
22
23
|
active_support/callbacks
|
@@ -61,18 +62,18 @@ module Karafka
|
|
61
62
|
end
|
62
63
|
|
63
64
|
# @return [String] path to a default file that contains booting procedure etc
|
64
|
-
# @note By default it is a file called '
|
65
|
-
# have Karafka that is merged into a Sinatra/Rails app and
|
65
|
+
# @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
|
66
|
+
# have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
|
66
67
|
# It will be used for console/workers/etc
|
67
68
|
# @example Standard only-Karafka case
|
68
|
-
# Karafka.boot_file #=> '/home/app_path/app.rb'
|
69
|
-
# @example Non standard case
|
70
|
-
# KARAFKA_BOOT_FILE='/home/app_path/karafka.rb'
|
71
69
|
# Karafka.boot_file #=> '/home/app_path/karafka.rb'
|
70
|
+
# @example Non standard case
|
71
|
+
# KARAFKA_BOOT_FILE='/home/app_path/app.rb'
|
72
|
+
# Karafka.boot_file #=> '/home/app_path/app.rb'
|
72
73
|
def boot_file
|
73
|
-
Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, '
|
74
|
+
Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
|
74
75
|
end
|
75
76
|
end
|
76
77
|
end
|
77
78
|
|
78
|
-
Karafka::Loader.
|
79
|
+
Karafka::Loader.load!(Karafka.core_root)
|
@@ -22,6 +22,7 @@ module Karafka
|
|
22
22
|
],
|
23
23
|
subscription: %i[start_from_beginning max_bytes_per_partition],
|
24
24
|
consuming: %i[min_bytes max_wait_time],
|
25
|
+
pausing: %i[pause_timeout],
|
25
26
|
# All the options that are under kafka config namespace, but are not used
|
26
27
|
# directly with kafka api, but from the Karafka user perspective, they are
|
27
28
|
# still related to kafka. They should not be proxied anywhere
|
@@ -32,7 +33,7 @@ module Karafka
|
|
32
33
|
# @return [Array<Symbol>] properties that can be set on a per topic level
|
33
34
|
def topic
|
34
35
|
(config_adapter[:subscription] + %i[
|
35
|
-
|
36
|
+
processing_backend
|
36
37
|
name
|
37
38
|
worker
|
38
39
|
parser
|
@@ -52,7 +53,7 @@ module Karafka
|
|
52
53
|
# only when proxying details go ruby-kafka. We use ignored fields internally in karafka
|
53
54
|
ignored_settings = config_adapter[:subscription]
|
54
55
|
defined_settings = config_adapter.values.flatten
|
55
|
-
karafka_settings = %i[batch_consuming
|
56
|
+
karafka_settings = %i[batch_consuming]
|
56
57
|
# This is a drity and bad hack of dry-configurable to get keys before setting values
|
57
58
|
dynamically_proxied = Karafka::Setup::Config
|
58
59
|
._settings
|
@@ -107,7 +107,14 @@ module Karafka
|
|
107
107
|
# will schedule a call task in sidekiq
|
108
108
|
def schedule
|
109
109
|
run_callbacks :schedule do
|
110
|
-
topic.
|
110
|
+
case topic.processing_backend
|
111
|
+
when :inline then
|
112
|
+
call_inline
|
113
|
+
when :sidekiq then
|
114
|
+
call_async
|
115
|
+
else
|
116
|
+
raise Errors::InvalidProcessingBackend, topic.processing_backend
|
117
|
+
end
|
111
118
|
end
|
112
119
|
end
|
113
120
|
|
data/lib/karafka/cli.rb
CHANGED
data/lib/karafka/cli/console.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
# Karafka framework Cli
|
5
|
-
class Cli
|
5
|
+
class Cli < Thor
|
6
6
|
# Console Karafka Cli action
|
7
7
|
class Console < Base
|
8
8
|
desc 'Start the Karafka console (short-cut alias: "c")'
|
@@ -22,7 +22,7 @@ module Karafka
|
|
22
22
|
# Start the Karafka console
|
23
23
|
def call
|
24
24
|
cli.info
|
25
|
-
|
25
|
+
exec self.class.command
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
data/lib/karafka/cli/flow.rb
CHANGED
data/lib/karafka/cli/info.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
# Karafka framework Cli
|
5
|
-
class Cli
|
5
|
+
class Cli < Thor
|
6
6
|
# Info Karafka Cli action
|
7
7
|
class Info < Base
|
8
8
|
desc 'Print configuration details and other options of your application'
|
@@ -14,7 +14,7 @@ module Karafka
|
|
14
14
|
info = [
|
15
15
|
"Karafka framework version: #{Karafka::VERSION}",
|
16
16
|
"Application client id: #{config.client_id}",
|
17
|
-
"
|
17
|
+
"Processing backend: #{config.processing_backend}",
|
18
18
|
"Batch consuming: #{config.batch_consuming}",
|
19
19
|
"Batch processing: #{config.batch_processing}",
|
20
20
|
"Number of threads: #{config.concurrency}",
|
data/lib/karafka/cli/install.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
# Karafka framework Cli
|
5
|
-
class Cli
|
5
|
+
class Cli < Thor
|
6
6
|
# Install Karafka Cli action
|
7
7
|
class Install < Base
|
8
8
|
desc 'Install all required things for Karafka application in current directory'
|
@@ -20,7 +20,7 @@ module Karafka
|
|
20
20
|
|
21
21
|
# Where should we map proper files from templates
|
22
22
|
INSTALL_FILES_MAP = {
|
23
|
-
'
|
23
|
+
'karafka.rb.example' => Karafka.boot_file.basename,
|
24
24
|
'sidekiq.yml.example' => 'config/sidekiq.yml.example',
|
25
25
|
'application_worker.rb.example' => 'app/workers/application_worker.rb',
|
26
26
|
'application_controller.rb.example' => 'app/controllers/application_controller.rb',
|
data/lib/karafka/cli/server.rb
CHANGED
@@ -2,27 +2,30 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
# Karafka framework Cli
|
5
|
-
class Cli
|
5
|
+
class Cli < Thor
|
6
6
|
# Server Karafka Cli action
|
7
7
|
class Server < Base
|
8
8
|
desc 'Start the Karafka server (short-cut alias: "s")'
|
9
9
|
option aliases: 's'
|
10
10
|
option :daemon, default: false, type: :boolean, aliases: :d
|
11
11
|
option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
|
12
|
+
option :consumer_groups, type: :array, default: nil, aliases: :g
|
12
13
|
|
13
14
|
# Start the Karafka server
|
14
15
|
def call
|
16
|
+
validate!
|
17
|
+
|
15
18
|
puts 'Starting Karafka server'
|
16
19
|
cli.info
|
17
20
|
|
18
21
|
if cli.options[:daemon]
|
22
|
+
FileUtils.mkdir_p File.dirname(cli.options[:pid])
|
19
23
|
# For some reason Celluloid spins threads that break forking
|
20
24
|
# Threads are not shutdown immediately so deamonization will stale until
|
21
25
|
# those threads are killed by Celluloid manager (via timeout)
|
22
26
|
# There's nothing initialized here yet, so instead we shutdown celluloid
|
23
27
|
# and run it again when we need (after fork)
|
24
28
|
Celluloid.shutdown
|
25
|
-
validate!
|
26
29
|
daemonize
|
27
30
|
Celluloid.boot
|
28
31
|
end
|
@@ -30,17 +33,22 @@ module Karafka
|
|
30
33
|
# Remove pidfile on shutdown
|
31
34
|
ObjectSpace.define_finalizer(String.new, proc { send(:clean) })
|
32
35
|
|
36
|
+
# We assign active topics on a server level, as only server is expected to listen on
|
37
|
+
# part of the topics
|
38
|
+
Karafka::Server.consumer_groups = cli.options[:consumer_groups]
|
39
|
+
|
33
40
|
# After we fork, we can boot celluloid again
|
34
41
|
Karafka::Server.run
|
35
42
|
end
|
36
43
|
|
37
44
|
private
|
38
45
|
|
39
|
-
#
|
40
|
-
#
|
46
|
+
# Checks the server cli configuration
|
47
|
+
# options validations in terms of app setup (topics, pid existence, etc)
|
41
48
|
def validate!
|
42
|
-
|
43
|
-
|
49
|
+
result = Schemas::ServerCliOptions.call(cli.options)
|
50
|
+
return if result.success?
|
51
|
+
raise Errors::InvalidConfiguration, result.errors
|
44
52
|
end
|
45
53
|
|
46
54
|
# Detaches current process into background and writes its pidfile
|
data/lib/karafka/cli/worker.rb
CHANGED
@@ -41,13 +41,7 @@ module Karafka
|
|
41
41
|
# @return [Hash] hash with all the settings required by Kafka#consumer method
|
42
42
|
def consumer(consumer_group)
|
43
43
|
settings = { group_id: consumer_group.id }
|
44
|
-
|
45
|
-
kafka_configs.each do |setting_name, setting_value|
|
46
|
-
next unless AttributesMap.config_adapter[:consumer].include?(setting_name)
|
47
|
-
next if settings.keys.include?(setting_name)
|
48
|
-
settings[setting_name] = setting_value
|
49
|
-
end
|
50
|
-
|
44
|
+
settings = fetch_for(:consumer, settings)
|
51
45
|
sanitize(settings)
|
52
46
|
end
|
53
47
|
|
@@ -57,15 +51,7 @@ module Karafka
|
|
57
51
|
# @return [Hash] hash with all the settings required by
|
58
52
|
# Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
|
59
53
|
def consuming(_consumer_group)
|
60
|
-
|
61
|
-
|
62
|
-
kafka_configs.each do |setting_name, setting_value|
|
63
|
-
next unless AttributesMap.config_adapter[:consuming].include?(setting_name)
|
64
|
-
next if settings.keys.include?(setting_name)
|
65
|
-
settings[setting_name] = setting_value
|
66
|
-
end
|
67
|
-
|
68
|
-
sanitize(settings)
|
54
|
+
sanitize(fetch_for(:consuming))
|
69
55
|
end
|
70
56
|
|
71
57
|
# Builds all the configuration settings for kafka consumer#subscribe method
|
@@ -73,18 +59,33 @@ module Karafka
|
|
73
59
|
# @return [Hash] hash with all the settings required by kafka consumer#subscribe method
|
74
60
|
def subscription(topic)
|
75
61
|
settings = { start_from_beginning: topic.start_from_beginning }
|
62
|
+
settings = fetch_for(:subscription, settings)
|
63
|
+
[Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
|
64
|
+
end
|
65
|
+
|
66
|
+
# Builds all the configuration settings required by kafka consumer#pause method
|
67
|
+
# @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
|
68
|
+
# @return [Hash] hash with all the settings required to pause kafka consumer
|
69
|
+
def pausing(consumer_group)
|
70
|
+
{ timeout: consumer_group.pause_timeout }
|
71
|
+
end
|
76
72
|
|
73
|
+
private
|
74
|
+
|
75
|
+
# Fetches proper settings for a given map namespace
|
76
|
+
# @param namespace_key [Symbol] namespace from attributes map config adapter hash
|
77
|
+
# @param preexisting_settings [Hash] hash with some preexisting settings that might have
|
78
|
+
# been loaded in a different way
|
79
|
+
def fetch_for(namespace_key, preexisting_settings = {})
|
77
80
|
kafka_configs.each do |setting_name, setting_value|
|
78
|
-
next unless AttributesMap.config_adapter[
|
79
|
-
next if
|
80
|
-
|
81
|
+
next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
|
82
|
+
next if preexisting_settings.keys.include?(setting_name)
|
83
|
+
preexisting_settings[setting_name] = setting_value
|
81
84
|
end
|
82
85
|
|
83
|
-
|
86
|
+
preexisting_settings
|
84
87
|
end
|
85
88
|
|
86
|
-
private
|
87
|
-
|
88
89
|
# Removes nil containing keys from the final settings so it can use Kafkas driver
|
89
90
|
# defaults for those
|
90
91
|
# @param settings [Hash] settings that may contain nil values
|
@@ -20,9 +20,19 @@ module Karafka
|
|
20
20
|
def fetch_loop
|
21
21
|
send(
|
22
22
|
consumer_group.batch_consuming ? :consume_each_batch : :consume_each_message
|
23
|
-
)
|
24
|
-
|
25
|
-
|
23
|
+
) { |messages| yield(messages) }
|
24
|
+
rescue Kafka::ProcessingError => e
|
25
|
+
# If there was an error during processing, we have to log it, pause current partition
|
26
|
+
# and process other things
|
27
|
+
Karafka.monitor.notice_error(self.class, e.cause)
|
28
|
+
pause(e.topic, e.partition)
|
29
|
+
retry
|
30
|
+
# This is on purpose - see the notes for this method
|
31
|
+
# rubocop:disable RescueException
|
32
|
+
rescue Exception => e
|
33
|
+
# rubocop:enable RescueException
|
34
|
+
Karafka.monitor.notice_error(self.class, e)
|
35
|
+
retry
|
26
36
|
end
|
27
37
|
|
28
38
|
# Gracefuly stops topic consumption
|
@@ -35,6 +45,16 @@ module Karafka
|
|
35
45
|
|
36
46
|
attr_reader :consumer_group
|
37
47
|
|
48
|
+
# Pauses processing of a given topic partition
|
49
|
+
# @param topic [String] topic that we want to pause
|
50
|
+
# @param partition [Integer] number partition that we want to pause
|
51
|
+
def pause(topic, partition)
|
52
|
+
settings = ConfigAdapter.pausing(consumer_group)
|
53
|
+
return false unless settings[:timeout].positive?
|
54
|
+
kafka_consumer.pause(topic, partition, settings)
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
38
58
|
# Consumes messages from Kafka in batches
|
39
59
|
# @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
|
40
60
|
def consume_each_batch
|
@@ -24,48 +24,29 @@ module Karafka
|
|
24
24
|
controller = Karafka::Routing::Router.build("#{group_id}_#{mapped_topic}")
|
25
25
|
handler = controller.topic.batch_processing ? :process_batch : :process_each
|
26
26
|
|
27
|
-
send(handler, controller,
|
28
|
-
# This is on purpose - see the notes for this method
|
29
|
-
# rubocop:disable RescueException
|
30
|
-
rescue Exception => e
|
31
|
-
# rubocop:enable RescueException
|
32
|
-
Karafka.monitor.notice_error(self, e)
|
27
|
+
send(handler, controller, kafka_messages)
|
33
28
|
end
|
34
29
|
|
35
30
|
private
|
36
31
|
|
37
32
|
# Processes whole batch in one request (all at once)
|
38
33
|
# @param controller [Karafka::BaseController] base controller descendant
|
39
|
-
# @param mapped_topic [String] mapped topic name
|
40
34
|
# @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
|
41
|
-
def process_batch(controller,
|
42
|
-
|
43
|
-
|
44
|
-
# we have to overwrite topic with our mapped topic version
|
45
|
-
# @note For the default mapper, it will be the same as topic
|
46
|
-
# @note We have to use instance_variable_set, as the Kafka::FetchedMessage does not
|
47
|
-
# provide attribute writers
|
48
|
-
kafka_message.instance_variable_set(:'@topic', mapped_topic)
|
49
|
-
kafka_message
|
50
|
-
end
|
51
|
-
|
52
|
-
controller.params_batch = messages_batch
|
53
|
-
|
54
|
-
Karafka.monitor.notice(self, messages_batch)
|
55
|
-
|
35
|
+
def process_batch(controller, kafka_messages)
|
36
|
+
controller.params_batch = kafka_messages
|
37
|
+
Karafka.monitor.notice(self, kafka_messages)
|
56
38
|
controller.schedule
|
57
39
|
end
|
58
40
|
|
59
41
|
# Processes messages one by one (like with std http requests)
|
60
42
|
# @param controller [Karafka::BaseController] base controller descendant
|
61
|
-
# @param mapped_topic [String] mapped topic name
|
62
43
|
# @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
|
63
|
-
def process_each(controller,
|
44
|
+
def process_each(controller, kafka_messages)
|
64
45
|
kafka_messages.each do |kafka_message|
|
65
46
|
# @note This is a simple trick - we just process one after another, but in order
|
66
47
|
# not to handle everywhere both cases (single vs batch), we just "fake" batching with
|
67
48
|
# a single message for each
|
68
|
-
process_batch(controller,
|
49
|
+
process_batch(controller, [kafka_message])
|
69
50
|
end
|
70
51
|
end
|
71
52
|
end
|
data/lib/karafka/errors.rb
CHANGED
@@ -41,5 +41,9 @@ module Karafka
|
|
41
41
|
# Raised when processing messages in batches but still want to use #params instead of
|
42
42
|
# #params_batch
|
43
43
|
ParamsMethodUnavailable = Class.new(BaseError)
|
44
|
+
|
45
|
+
# Raised when for some reason we try to use invalid processing backend and
|
46
|
+
# we bypass validations
|
47
|
+
InvalidProcessingBackend = Class.new(BaseError)
|
44
48
|
end
|
45
49
|
end
|
data/lib/karafka/fetcher.rb
CHANGED
@@ -26,7 +26,7 @@ module Karafka
|
|
26
26
|
|
27
27
|
# @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
|
28
28
|
def listeners
|
29
|
-
@listeners ||= App.consumer_groups.map do |consumer_group|
|
29
|
+
@listeners ||= App.consumer_groups.active.map do |consumer_group|
|
30
30
|
Karafka::Connection::Listener.new(consumer_group)
|
31
31
|
end
|
32
32
|
end
|
data/lib/karafka/loader.rb
CHANGED
@@ -2,78 +2,28 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
# Loader for requiring all the files in a proper order
|
5
|
-
|
6
|
-
# try to figure it out. It will load 'base*' files first and then
|
7
|
-
# any other.
|
8
|
-
class Loader
|
5
|
+
module Loader
|
9
6
|
# Order in which we want to load app files
|
10
7
|
DIRS = %w[
|
11
8
|
config/initializers
|
12
9
|
lib
|
13
|
-
app/helpers
|
14
|
-
app/inputs
|
15
|
-
app/decorators
|
16
|
-
app/models/concerns
|
17
|
-
app/models
|
18
|
-
app/responders
|
19
|
-
app/services
|
20
|
-
app/presenters
|
21
|
-
app/workers
|
22
|
-
app/controllers
|
23
|
-
app/aspects
|
24
10
|
app
|
25
11
|
].freeze
|
26
12
|
|
27
13
|
# Will load files in a proper order (based on DIRS)
|
28
14
|
# @param [String] root path from which we want to start
|
29
|
-
def load(root)
|
15
|
+
def self.load(root)
|
30
16
|
DIRS.each do |dir|
|
31
17
|
path = File.join(root, dir)
|
18
|
+
next unless File.exist?(path)
|
32
19
|
load!(path)
|
33
20
|
end
|
34
21
|
end
|
35
22
|
|
36
23
|
# Requires all the ruby files from one path in a proper order
|
37
24
|
# @param path [String] path (dir) from which we want to load ruby files in a proper order
|
38
|
-
|
39
|
-
|
40
|
-
base_load!(path)
|
41
|
-
files_load!(path)
|
42
|
-
end
|
43
|
-
|
44
|
-
# Requires all the ruby files from one relative path inside application directory
|
45
|
-
# @param relative_path [String] relative path (dir) to a file inside application directory
|
46
|
-
# from which we want to load ruby files in a proper order
|
47
|
-
def relative_load!(relative_path)
|
48
|
-
path = File.join(::Karafka.root, relative_path)
|
49
|
-
load!(path)
|
50
|
-
end
|
51
|
-
|
52
|
-
private
|
53
|
-
|
54
|
-
# Loads all the base files
|
55
|
-
# @param path [String] path (dir) from which we want to load ruby base files in a proper order
|
56
|
-
def base_load!(path)
|
57
|
-
bases = File.join(path, '**/base*.rb')
|
58
|
-
Dir[bases].sort(&method(:base_sorter)).each(&method(:require))
|
59
|
-
end
|
60
|
-
|
61
|
-
# Loads all other files (not base)
|
62
|
-
# @param path [String] path (dir) from which we want to load ruby files in a proper order
|
63
|
-
# @note Technically it will load the base files again but they are already loaded so nothing
|
64
|
-
# will happen
|
65
|
-
def files_load!(path)
|
66
|
-
files = File.join(path, '**/*.rb')
|
67
|
-
Dir[files].sort.each(&method(:require))
|
68
|
-
end
|
69
|
-
|
70
|
-
# @return [Integer] order for sorting
|
71
|
-
# @note We need sort all base files based on their position in a file tree
|
72
|
-
# so all the files that are "higher" should be loaded first
|
73
|
-
# @param str1 [String] first string for comparison
|
74
|
-
# @param str2 [String] second string for comparison
|
75
|
-
def base_sorter(str1, str2)
|
76
|
-
str1.count('/') <=> str2.count('/')
|
25
|
+
def self.load!(path)
|
26
|
+
require_all(path)
|
77
27
|
end
|
78
28
|
end
|
79
29
|
end
|
data/lib/karafka/monitor.rb
CHANGED
@@ -10,7 +10,6 @@ module Karafka
|
|
10
10
|
class Params < HashWithIndifferentAccess
|
11
11
|
# Kafka::FetchedMessage attributes that we want to use inside of params
|
12
12
|
KAFKA_MESSAGE_ATTRIBUTES = %i[
|
13
|
-
topic
|
14
13
|
value
|
15
14
|
partition
|
16
15
|
offset
|
@@ -44,6 +43,11 @@ module Karafka
|
|
44
43
|
KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
|
45
44
|
instance[attribute] = message.send(attribute)
|
46
45
|
end
|
46
|
+
|
47
|
+
# When we get raw messages, they might have a topic, that was modified by a
|
48
|
+
# topic mapper. We need to "reverse" this change and map back to the non-modified
|
49
|
+
# format, so our internal flow is not corrupted with the mapping
|
50
|
+
instance[:topic] = Karafka::App.config.topic_mapper.incoming(message.topic)
|
47
51
|
end
|
48
52
|
end
|
49
53
|
end
|
data/lib/karafka/parsers/json.rb
CHANGED
@@ -10,8 +10,8 @@ module Karafka
|
|
10
10
|
# @example
|
11
11
|
# Json.parse("{\"a\":1}") #=> { 'a' => 1 }
|
12
12
|
def self.parse(content)
|
13
|
-
::
|
14
|
-
rescue ::
|
13
|
+
::MultiJson.load(content)
|
14
|
+
rescue ::MultiJson::ParseError => e
|
15
15
|
raise ::Karafka::Errors::ParserError, e
|
16
16
|
end
|
17
17
|
|
@@ -32,6 +32,13 @@ module Karafka
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
35
|
+
# @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
|
36
|
+
# we want to use. Since Karafka supports multi-process setup, we need to be able
|
37
|
+
# to pick only those consumer groups that should be active in our given process context
|
38
|
+
def active
|
39
|
+
select(&:active?)
|
40
|
+
end
|
41
|
+
|
35
42
|
private
|
36
43
|
|
37
44
|
# Builds and saves given consumer group
|
@@ -10,16 +10,23 @@ module Karafka
|
|
10
10
|
|
11
11
|
attr_reader :topics
|
12
12
|
attr_reader :id
|
13
|
+
attr_reader :name
|
13
14
|
|
14
|
-
# @param
|
15
|
+
# @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
|
15
16
|
# yet have an application client_id namespace, this will be added here by default.
|
16
17
|
# We add it to make a multi-system development easier for people that don't use
|
17
18
|
# kafka and don't understand the concept of consumer groups.
|
18
|
-
def initialize(
|
19
|
-
@
|
19
|
+
def initialize(name)
|
20
|
+
@name = name
|
21
|
+
@id = "#{Karafka::App.config.client_id.to_s.underscore}_#{@name}"
|
20
22
|
@topics = []
|
21
23
|
end
|
22
24
|
|
25
|
+
# @return [Boolean] true if this consumer group should be active in our current process
|
26
|
+
def active?
|
27
|
+
Karafka::Server.consumer_groups.include?(name)
|
28
|
+
end
|
29
|
+
|
23
30
|
# Builds a topic representation inside of a current consumer group route
|
24
31
|
# @param name [String, Symbol] name of topic to which we want to subscribe
|
25
32
|
# @yield Evaluates a given block in a topic context
|
@@ -36,7 +36,7 @@ module Karafka
|
|
36
36
|
# background job
|
37
37
|
# @note If not provided - will be built based on the provided controller
|
38
38
|
def worker
|
39
|
-
@worker ||=
|
39
|
+
@worker ||= processing_backend == :sidekiq ? Workers::Builder.new(controller).build : nil
|
40
40
|
end
|
41
41
|
|
42
42
|
# @return [Class, nil] Class (not an instance) of a responder that should respond from
|
@@ -20,11 +20,11 @@ module Karafka
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
optional(:
|
23
|
+
optional(:processing_backend).filled(included_in?: %i[inline sidekiq])
|
24
24
|
|
25
|
-
# If
|
26
|
-
rule(redis_presence: %i[redis
|
27
|
-
|
25
|
+
# If we want to use sidekiq, then redis needs to be configured
|
26
|
+
rule(redis_presence: %i[redis processing_backend]) do |redis, processing_backend|
|
27
|
+
processing_backend.eql?(:sidekiq).then(redis.filled?)
|
28
28
|
end
|
29
29
|
|
30
30
|
optional(:connection_pool).schema do
|
@@ -6,7 +6,7 @@ module Karafka
|
|
6
6
|
ConsumerGroupTopic = Dry::Validation.Schema do
|
7
7
|
required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
8
8
|
required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
9
|
-
required(:
|
9
|
+
required(:processing_backend).filled(included_in?: %i[inline sidekiq])
|
10
10
|
required(:controller).filled
|
11
11
|
required(:parser).filled
|
12
12
|
required(:interchanger).filled
|
@@ -20,11 +20,11 @@ module Karafka
|
|
20
20
|
required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
21
21
|
required(:seed_brokers).filled(:array?)
|
22
22
|
required(:session_timeout).filled(:int?)
|
23
|
+
required(:pause_timeout).filled(:int?, gteq?: 0)
|
23
24
|
required(:offset_commit_interval).filled(:int?)
|
24
25
|
required(:offset_commit_threshold).filled(:int?)
|
25
26
|
required(:offset_retention_time) { none?.not > int? }
|
26
27
|
required(:heartbeat_interval).filled(:int?, gteq?: 0)
|
27
|
-
required(:topic_mapper).filled
|
28
28
|
required(:connect_timeout).filled(:int?, gt?: 0)
|
29
29
|
required(:socket_timeout).filled(:int?, gt?: 0)
|
30
30
|
required(:max_wait_time).filled(:int?, gteq?: 0)
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Schemas
|
5
|
+
# Schema for validating correctness of the server cli command options
|
6
|
+
# We validate some basics + the list of consumer_groups on which we want to use, to make
|
7
|
+
# sure that all of them are defined, plus that a pidfile does not exist
|
8
|
+
ServerCliOptions = Dry::Validation.Schema do
|
9
|
+
configure do
|
10
|
+
option :consumer_groups
|
11
|
+
|
12
|
+
def self.messages
|
13
|
+
super.merge(
|
14
|
+
en: {
|
15
|
+
errors: {
|
16
|
+
consumer_groups_inclusion: 'Unknown consumer group.',
|
17
|
+
pid_existence: 'Pidfile already exists.'
|
18
|
+
}
|
19
|
+
}
|
20
|
+
)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
optional(:pid).filled(:str?)
|
25
|
+
optional(:daemon).filled(:bool?)
|
26
|
+
optional(:consumer_groups).filled(:array?)
|
27
|
+
|
28
|
+
validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
|
29
|
+
# If there were no consumer_groups declared in the server cli, it means that we will
|
30
|
+
# run all of them and no need to validate them here at all
|
31
|
+
if consumer_groups.nil?
|
32
|
+
true
|
33
|
+
else
|
34
|
+
(consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
validate(pid_existence: :pid) do |pid|
|
39
|
+
pid ? !File.exist?(pid) : true
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
data/lib/karafka/server.rb
CHANGED
@@ -8,6 +8,9 @@ module Karafka
|
|
8
8
|
# So we can have access to them later on and be able to stop them on exit
|
9
9
|
attr_reader :consumers
|
10
10
|
|
11
|
+
# Writer for list of consumer groups that we want to consume in our current process context
|
12
|
+
attr_writer :consumer_groups
|
13
|
+
|
11
14
|
# Method which runs app
|
12
15
|
def run
|
13
16
|
@consumers = Concurrent::Array.new
|
@@ -17,6 +20,13 @@ module Karafka
|
|
17
20
|
start_supervised
|
18
21
|
end
|
19
22
|
|
23
|
+
# @return [Array<String>] array with names of consumer groups that should be consumed in a
|
24
|
+
# current server context
|
25
|
+
def consumer_groups
|
26
|
+
# If not specified, a server will listed on all the topics
|
27
|
+
@consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
|
28
|
+
end
|
29
|
+
|
20
30
|
private
|
21
31
|
|
22
32
|
# @return [Karafka::Process] process wrapper instance used to catch system signal calls
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -18,9 +18,9 @@ module Karafka
|
|
18
18
|
# option client_id [String] kafka client_id - used to provide
|
19
19
|
# default Kafka groups namespaces and identify that app in kafka
|
20
20
|
setting :client_id
|
21
|
-
#
|
22
|
-
#
|
23
|
-
setting :
|
21
|
+
# How should we process messages. For now we support inline mode (asap in the process) or
|
22
|
+
# sidekiq mode (schedule to sidekiq)
|
23
|
+
setting :processing_backend, :inline
|
24
24
|
# option logger [Instance] logger that we want to use
|
25
25
|
setting :logger, -> { ::Karafka::Logger.instance }
|
26
26
|
# option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
|
@@ -51,7 +51,7 @@ module Karafka
|
|
51
51
|
# or the opposite for bigger systems
|
52
52
|
setting :size, lambda {
|
53
53
|
[
|
54
|
-
::Karafka::App.consumer_groups.count,
|
54
|
+
::Karafka::App.consumer_groups.active.count,
|
55
55
|
Sidekiq.options[:concurrency]
|
56
56
|
].max
|
57
57
|
}
|
@@ -67,6 +67,11 @@ module Karafka
|
|
67
67
|
# option session_timeout [Integer] the number of seconds after which, if a client
|
68
68
|
# hasn't contacted the Kafka cluster, it will be kicked out of the group.
|
69
69
|
setting :session_timeout, 30
|
70
|
+
# Time that a given partition will be paused from processing messages, when message
|
71
|
+
# processing fails. It allows us to process other partitions, while the error is being
|
72
|
+
# resolved and also "slows" things down, so it prevents from "eating" up all messages and
|
73
|
+
# processing them with failed code
|
74
|
+
setting :pause_timeout, 10
|
70
75
|
# option offset_commit_interval [Integer] the interval between offset commits,
|
71
76
|
# in seconds.
|
72
77
|
setting :offset_commit_interval, 10
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Application worker from which all workers should inherit
|
2
4
|
# You can rename it if it would conflict with your current code base (in case you're integrating
|
3
5
|
# Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker
|
@@ -1,19 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Non Ruby on Rails setup
|
1
4
|
ENV['RACK_ENV'] ||= 'development'
|
2
5
|
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
3
|
-
|
4
6
|
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
7
|
+
Karafka::Loader.load(Karafka::App.root)
|
5
8
|
|
6
|
-
|
9
|
+
# Ruby on Rails setup
|
10
|
+
# Remove whole non-Rails setup that is above and uncomment the 4 lines below
|
11
|
+
# ENV['RAILS_ENV'] ||= 'development'
|
12
|
+
# ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
|
13
|
+
# require ::File.expand_path('../config/environment', __FILE__)
|
14
|
+
# Rails.application.eager_load!
|
7
15
|
|
8
|
-
|
9
|
-
class App < Karafka::App
|
16
|
+
class KarafkaApp < Karafka::App
|
10
17
|
setup do |config|
|
11
18
|
config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
|
12
19
|
config.client_id = 'example_app'
|
13
20
|
config.redis = {
|
14
21
|
url: 'redis://localhost:6379'
|
15
22
|
}
|
16
|
-
config.
|
23
|
+
config.processing_backend = Karafka.env.development? ? :inline : :sidekiq
|
17
24
|
config.batch_consuming = true
|
18
25
|
end
|
19
26
|
|
@@ -35,4 +42,4 @@ class App < Karafka::App
|
|
35
42
|
end
|
36
43
|
end
|
37
44
|
|
38
|
-
|
45
|
+
KarafkaApp.boot!
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.0.
|
4
|
+
version: 0.6.0.rc2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-08-
|
13
|
+
date: 2017-08-23 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: ruby-kafka
|
@@ -153,75 +153,33 @@ dependencies:
|
|
153
153
|
- !ruby/object:Gem::Version
|
154
154
|
version: '0.7'
|
155
155
|
- !ruby/object:Gem::Dependency
|
156
|
-
name:
|
156
|
+
name: multi_json
|
157
157
|
requirement: !ruby/object:Gem::Requirement
|
158
158
|
requirements:
|
159
159
|
- - ">="
|
160
160
|
- !ruby/object:Gem::Version
|
161
|
-
version: 1.
|
161
|
+
version: '1.12'
|
162
162
|
type: :runtime
|
163
163
|
prerelease: false
|
164
164
|
version_requirements: !ruby/object:Gem::Requirement
|
165
165
|
requirements:
|
166
166
|
- - ">="
|
167
167
|
- !ruby/object:Gem::Version
|
168
|
-
version: 1.
|
168
|
+
version: '1.12'
|
169
169
|
- !ruby/object:Gem::Dependency
|
170
|
-
name:
|
170
|
+
name: require_all
|
171
171
|
requirement: !ruby/object:Gem::Requirement
|
172
|
-
requirements:
|
173
|
-
- - "~>"
|
174
|
-
- !ruby/object:Gem::Version
|
175
|
-
version: '1.2'
|
176
|
-
type: :development
|
177
|
-
prerelease: false
|
178
|
-
version_requirements: !ruby/object:Gem::Requirement
|
179
|
-
requirements:
|
180
|
-
- - "~>"
|
181
|
-
- !ruby/object:Gem::Version
|
182
|
-
version: '1.2'
|
183
|
-
- !ruby/object:Gem::Dependency
|
184
|
-
name: rspec
|
185
|
-
requirement: !ruby/object:Gem::Requirement
|
186
|
-
requirements:
|
187
|
-
- - ">="
|
188
|
-
- !ruby/object:Gem::Version
|
189
|
-
version: '3.6'
|
190
|
-
type: :development
|
191
|
-
prerelease: false
|
192
|
-
version_requirements: !ruby/object:Gem::Requirement
|
193
172
|
requirements:
|
194
173
|
- - ">="
|
195
174
|
- !ruby/object:Gem::Version
|
196
|
-
version: '
|
197
|
-
|
198
|
-
name: simplecov
|
199
|
-
requirement: !ruby/object:Gem::Requirement
|
200
|
-
requirements:
|
201
|
-
- - ">="
|
202
|
-
- !ruby/object:Gem::Version
|
203
|
-
version: '0.14'
|
204
|
-
type: :development
|
205
|
-
prerelease: false
|
206
|
-
version_requirements: !ruby/object:Gem::Requirement
|
207
|
-
requirements:
|
208
|
-
- - ">="
|
209
|
-
- !ruby/object:Gem::Version
|
210
|
-
version: '0.14'
|
211
|
-
- !ruby/object:Gem::Dependency
|
212
|
-
name: byebug
|
213
|
-
requirement: !ruby/object:Gem::Requirement
|
214
|
-
requirements:
|
215
|
-
- - ">="
|
216
|
-
- !ruby/object:Gem::Version
|
217
|
-
version: '0'
|
218
|
-
type: :development
|
175
|
+
version: '1.4'
|
176
|
+
type: :runtime
|
219
177
|
prerelease: false
|
220
178
|
version_requirements: !ruby/object:Gem::Requirement
|
221
179
|
requirements:
|
222
180
|
- - ">="
|
223
181
|
- !ruby/object:Gem::Version
|
224
|
-
version: '
|
182
|
+
version: '1.4'
|
225
183
|
description: Framework used to simplify Apache Kafka based Ruby applications development
|
226
184
|
email:
|
227
185
|
- maciej@coditsu.io
|
@@ -292,6 +250,7 @@ files:
|
|
292
250
|
- lib/karafka/schemas/config.rb
|
293
251
|
- lib/karafka/schemas/consumer_group.rb
|
294
252
|
- lib/karafka/schemas/responder_usage.rb
|
253
|
+
- lib/karafka/schemas/server_cli_options.rb
|
295
254
|
- lib/karafka/server.rb
|
296
255
|
- lib/karafka/setup/config.rb
|
297
256
|
- lib/karafka/setup/configurators/base.rb
|
@@ -299,10 +258,10 @@ files:
|
|
299
258
|
- lib/karafka/setup/configurators/sidekiq.rb
|
300
259
|
- lib/karafka/setup/configurators/water_drop.rb
|
301
260
|
- lib/karafka/status.rb
|
302
|
-
- lib/karafka/templates/app.rb.example
|
303
261
|
- lib/karafka/templates/application_controller.rb.example
|
304
262
|
- lib/karafka/templates/application_responder.rb.example
|
305
263
|
- lib/karafka/templates/application_worker.rb.example
|
264
|
+
- lib/karafka/templates/karafka.rb.example
|
306
265
|
- lib/karafka/templates/sidekiq.yml.example
|
307
266
|
- lib/karafka/version.rb
|
308
267
|
- lib/karafka/workers/builder.rb
|