karafka 1.2.0 → 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.coditsu/ci.yml +3 -0
- data/.console_irbrc +1 -3
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +35 -17
- data/CHANGELOG.md +113 -0
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +3 -2
- data/Gemfile.lock +85 -71
- data/README.md +20 -8
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/karafka.gemspec +18 -11
- data/lib/karafka.rb +8 -15
- data/lib/karafka/app.rb +14 -6
- data/lib/karafka/attributes_map.rb +15 -14
- data/lib/karafka/base_consumer.rb +19 -30
- data/lib/karafka/base_responder.rb +51 -29
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +0 -1
- data/lib/karafka/cli/info.rb +3 -1
- data/lib/karafka/cli/install.rb +30 -6
- data/lib/karafka/cli/server.rb +11 -6
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb} +54 -19
- data/lib/karafka/connection/batch_delegator.rb +51 -0
- data/lib/karafka/connection/builder.rb +16 -0
- data/lib/karafka/connection/client.rb +40 -40
- data/lib/karafka/connection/listener.rb +26 -15
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/callbacks.rb +32 -15
- data/lib/karafka/consumers/includer.rb +30 -18
- data/lib/karafka/consumers/metadata.rb +10 -0
- data/lib/karafka/consumers/responders.rb +2 -2
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +29 -0
- data/lib/karafka/errors.rb +18 -17
- data/lib/karafka/fetcher.rb +28 -30
- data/lib/karafka/helpers/class_matcher.rb +11 -1
- data/lib/karafka/helpers/config_retriever.rb +1 -1
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +9 -6
- data/lib/karafka/instrumentation/monitor.rb +15 -9
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
- data/lib/karafka/params/builders/metadata.rb +33 -0
- data/lib/karafka/params/builders/params.rb +36 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +35 -0
- data/lib/karafka/params/params.rb +68 -0
- data/lib/karafka/params/params_batch.rb +35 -20
- data/lib/karafka/patches/ruby_kafka.rb +21 -8
- data/lib/karafka/persistence/client.rb +15 -11
- data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +0 -4
- data/lib/karafka/responders/builder.rb +1 -1
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +36 -8
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +9 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/topic.rb +5 -3
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +30 -41
- data/lib/karafka/setup/config.rb +72 -40
- data/lib/karafka/setup/configurators/water_drop.rb +8 -4
- data/lib/karafka/setup/dsl.rb +0 -1
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +95 -60
- metadata.gz.sig +0 -0
- data/lib/karafka/callbacks.rb +0 -30
- data/lib/karafka/callbacks/config.rb +0 -22
- data/lib/karafka/callbacks/dsl.rb +0 -16
- data/lib/karafka/connection/delegator.rb +0 -46
- data/lib/karafka/instrumentation/listener.rb +0 -112
- data/lib/karafka/loader.rb +0 -28
- data/lib/karafka/params/dsl.rb +0 -156
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -35
- data/lib/karafka/persistence/topic.rb +0 -29
- data/lib/karafka/schemas/config.rb +0 -24
- data/lib/karafka/schemas/consumer_group.rb +0 -77
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -29
- data/lib/karafka/setup/configurators/params.rb +0 -25
- data/lib/karafka/templates/karafka.rb.example +0 -54
data/README.md
CHANGED
@@ -2,21 +2,27 @@
|
|
2
2
|
|
3
3
|
[![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
|
4
4
|
|
5
|
+
**Note**: Documentation presented here refers to Karafka `1.3.0`.
|
6
|
+
|
7
|
+
If you're upgrading from `1.2.0`, please refer to our [Upgrade Notes article](https://mensfeld.pl/2019/09/karafka-framework-1-3-0-release-notes-ruby-kafka/).
|
8
|
+
|
9
|
+
If you are looking for the documentation for Karafka `1.2.*`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
|
10
|
+
|
11
|
+
## About Karafka
|
12
|
+
|
5
13
|
Framework used to simplify Apache Kafka based Ruby applications development.
|
6
14
|
|
7
15
|
Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming and processing this data, without having to focus on things that are not your business domain.
|
8
16
|
|
9
17
|
Karafka not only handles incoming messages but also provides tools for building complex data-flow applications that receive and send messages.
|
10
18
|
|
11
|
-
**Warning**: Wiki and all the docs refer to the 1.2.0.beta4. Sorry for the inconvenience. We will release the stable 1.2.0 version soon.
|
12
|
-
|
13
19
|
## How does it work
|
14
20
|
|
15
|
-
Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications
|
21
|
+
Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similar to how Rails applications are being built.
|
16
22
|
|
17
23
|
### Some things you might wonder about:
|
18
24
|
|
19
|
-
- You can integrate Karafka with **any** Ruby
|
25
|
+
- You can integrate Karafka with **any** Ruby-based application.
|
20
26
|
- Karafka does **not** require Sidekiq or any other third party software (apart from Kafka itself).
|
21
27
|
- Karafka works with Ruby on Rails but it is a **standalone** framework that can work without it.
|
22
28
|
- Karafka has a **minimal** set of dependencies, so adding it won't be a huge burden for your already existing applications.
|
@@ -27,19 +33,25 @@ Karafka based applications can be easily deployed to any type of infrastructure,
|
|
27
33
|
* Heroku
|
28
34
|
* Capistrano
|
29
35
|
* Docker
|
36
|
+
* Terraform
|
30
37
|
|
31
38
|
## Support
|
32
39
|
|
33
|
-
Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
|
40
|
+
Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
|
34
41
|
|
35
42
|
If you have any questions about using Karafka, feel free to join our [Gitter](https://gitter.im/karafka/karafka) chat channel.
|
36
43
|
|
37
44
|
## Getting started
|
38
45
|
|
46
|
+
If you're completely new to the subject, you can start with our "Kafka on Rails" articles series, that will get you up and running with the terminology and basic ideas behind using Kafka:
|
47
|
+
|
48
|
+
- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
|
49
|
+
- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Ruby and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
|
50
|
+
|
39
51
|
If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
|
40
52
|
|
41
53
|
```bash
|
42
|
-
git clone https://github.com/karafka/
|
54
|
+
git clone https://github.com/karafka/example-app ./example_app
|
43
55
|
```
|
44
56
|
|
45
57
|
then, just bundle install all the dependencies:
|
@@ -49,7 +61,7 @@ cd ./example_app
|
|
49
61
|
bundle install
|
50
62
|
```
|
51
63
|
|
52
|
-
and follow the instructions from the [example app Wiki](https://github.com/karafka/
|
64
|
+
and follow the instructions from the [example app Wiki](https://github.com/karafka/example-app/blob/master/README.md).
|
53
65
|
|
54
66
|
**Note**: you need to ensure, that you have Kafka up and running and you need to configure Kafka seed_brokers in the ```karafka.rb``` file.
|
55
67
|
|
@@ -69,7 +81,7 @@ Karafka framework and Karafka team are __not__ related to Kafka streaming servic
|
|
69
81
|
|
70
82
|
First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
|
71
83
|
|
72
|
-
Each pull request must pass all the
|
84
|
+
Each pull request must pass all the RSpec specs and meet our quality requirements.
|
73
85
|
|
74
86
|
To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
|
75
87
|
|
data/bin/karafka
CHANGED
@@ -10,7 +10,7 @@ else
|
|
10
10
|
# However when it is unavailable, we still want to be able to run help command
|
11
11
|
# and install command as they don't require configured app itself to run
|
12
12
|
raise(
|
13
|
-
Karafka::Errors::
|
13
|
+
Karafka::Errors::MissingBootFileError,
|
14
14
|
Karafka.boot_file
|
15
15
|
) unless %w[-h install].include?(ARGV[0])
|
16
16
|
end
|
data/certs/mensfeld.pem
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
3
|
+
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
|
4
|
+
NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
5
|
+
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
|
6
|
+
dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
|
7
|
+
uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
|
8
|
+
MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
|
9
|
+
y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
|
10
|
+
seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
|
11
|
+
94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
|
12
|
+
TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
|
13
|
+
Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
14
|
+
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
|
15
|
+
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
16
|
+
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
|
17
|
+
ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
|
18
|
+
xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
|
19
|
+
A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
|
20
|
+
hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
|
21
|
+
CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
|
22
|
+
i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
|
23
|
+
KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
|
24
|
+
nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
|
25
|
+
-----END CERTIFICATE-----
|
data/config/errors.yml
CHANGED
@@ -1,6 +1,39 @@
|
|
1
1
|
en:
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
dry_validation:
|
3
|
+
errors:
|
4
|
+
invalid_broker_schema: >
|
5
|
+
has an invalid format
|
6
|
+
Expected schema, host and port number
|
7
|
+
Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
|
8
|
+
invalid_certificate: >
|
9
|
+
is not a valid certificate
|
10
|
+
invalid_certificate_from_path: >
|
11
|
+
is not a valid certificate
|
12
|
+
invalid_private_key: >
|
13
|
+
is not a valid private key
|
14
|
+
max_timeout_size_for_exponential: >
|
15
|
+
pause_timeout cannot be more than pause_max_timeout
|
16
|
+
max_wait_time_limit:
|
17
|
+
max_wait_time cannot be more than socket_timeout
|
18
|
+
topics_names_not_unique: >
|
19
|
+
all topic names within a single consumer group must be unique
|
20
|
+
ssl_client_cert_with_ssl_client_cert_key: >
|
21
|
+
Both ssl_client_cert and ssl_client_cert_key need to be provided
|
22
|
+
ssl_client_cert_key_with_ssl_client_cert: >
|
23
|
+
Both ssl_client_cert_key and ssl_client_cert need to be provided
|
24
|
+
ssl_client_cert_chain_with_ssl_client_cert: >
|
25
|
+
Both ssl_client_cert_chain and ssl_client_cert need to be provided
|
26
|
+
ssl_client_cert_chain_with_ssl_client_cert_key: >
|
27
|
+
Both ssl_client_cert_chain and ssl_client_cert_key need to be provided
|
28
|
+
ssl_client_cert_key_password_with_ssl_client_cert_key: >
|
29
|
+
Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided
|
30
|
+
does_not_respond_to_token: >
|
31
|
+
needs to respond to a #token method
|
32
|
+
required_usage_count: >
|
33
|
+
Given topic must be used at least once
|
34
|
+
pid_already_exists: >
|
35
|
+
Pidfile already exists
|
36
|
+
consumer_groups_inclusion: >
|
37
|
+
Unknown consumer group
|
38
|
+
does_not_exist:
|
39
|
+
Given file does not exist or cannot be read
|
data/karafka.gemspec
CHANGED
@@ -5,6 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
require 'karafka/version'
|
7
7
|
|
8
|
+
# rubocop:disable Metrics/BlockLength
|
8
9
|
Gem::Specification.new do |spec|
|
9
10
|
spec.name = 'karafka'
|
10
11
|
spec.version = ::Karafka::VERSION
|
@@ -16,22 +17,28 @@ Gem::Specification.new do |spec|
|
|
16
17
|
spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
|
17
18
|
spec.license = 'MIT'
|
18
19
|
|
19
|
-
spec.add_dependency '
|
20
|
-
spec.add_dependency 'dry-
|
21
|
-
spec.add_dependency 'dry-
|
22
|
-
spec.add_dependency 'dry-
|
23
|
-
spec.add_dependency '
|
24
|
-
spec.add_dependency '
|
20
|
+
spec.add_dependency 'dry-configurable', '~> 0.8'
|
21
|
+
spec.add_dependency 'dry-inflector', '~> 0.1'
|
22
|
+
spec.add_dependency 'dry-monitor', '~> 0.3'
|
23
|
+
spec.add_dependency 'dry-validation', '~> 1.2'
|
24
|
+
spec.add_dependency 'envlogic', '~> 1.1'
|
25
|
+
spec.add_dependency 'irb', '~> 1.0'
|
25
26
|
spec.add_dependency 'multi_json', '>= 1.12'
|
26
27
|
spec.add_dependency 'rake', '>= 11.3'
|
27
|
-
spec.add_dependency '
|
28
|
-
spec.add_dependency '
|
29
|
-
spec.add_dependency '
|
30
|
-
spec.add_dependency '
|
28
|
+
spec.add_dependency 'ruby-kafka', '>= 0.7.8'
|
29
|
+
spec.add_dependency 'thor', '~> 0.20'
|
30
|
+
spec.add_dependency 'waterdrop', '~> 1.3.0'
|
31
|
+
spec.add_dependency 'zeitwerk', '~> 2.1'
|
31
32
|
|
32
|
-
spec.required_ruby_version = '>= 2.
|
33
|
+
spec.required_ruby_version = '>= 2.5.0'
|
33
34
|
|
35
|
+
if $PROGRAM_NAME.end_with?('gem')
|
36
|
+
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
37
|
+
end
|
38
|
+
|
39
|
+
spec.cert_chain = %w[certs/mensfeld.pem]
|
34
40
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
35
41
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
36
42
|
spec.require_paths = %w[lib]
|
37
43
|
end
|
44
|
+
# rubocop:enable Metrics/BlockLength
|
data/lib/karafka.rb
CHANGED
@@ -6,15 +6,16 @@
|
|
6
6
|
kafka
|
7
7
|
envlogic
|
8
8
|
thor
|
9
|
+
forwardable
|
9
10
|
fileutils
|
10
11
|
multi_json
|
11
|
-
require_all
|
12
12
|
dry-configurable
|
13
13
|
dry-validation
|
14
|
+
dry/events/publisher
|
14
15
|
dry/inflector
|
15
16
|
dry/monitor/notifications
|
16
|
-
|
17
|
-
|
17
|
+
dry/core/constants
|
18
|
+
zeitwerk
|
18
19
|
].each(&method(:require))
|
19
20
|
|
20
21
|
# Karafka library
|
@@ -62,17 +63,9 @@ module Karafka
|
|
62
63
|
end
|
63
64
|
end
|
64
65
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
setup/config
|
70
|
-
status
|
71
|
-
schemas/config
|
72
|
-
schemas/consumer_group_topic
|
73
|
-
schemas/consumer_group
|
74
|
-
].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
|
66
|
+
Zeitwerk::Loader
|
67
|
+
.for_gem
|
68
|
+
.tap(&:setup)
|
69
|
+
.tap(&:eager_load)
|
75
70
|
|
76
|
-
Karafka::Loader.load!(Karafka.core_root)
|
77
71
|
Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
|
78
|
-
Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)
|
data/lib/karafka/app.rb
CHANGED
@@ -4,28 +4,36 @@ module Karafka
|
|
4
4
|
# App class
|
5
5
|
class App
|
6
6
|
extend Setup::Dsl
|
7
|
-
extend Callbacks::Dsl
|
8
7
|
|
9
8
|
class << self
|
10
9
|
# Sets up all the internal components and bootstrap whole app
|
11
10
|
# We need to know details about consumers in order to setup components,
|
12
11
|
# that's why we don't setup them after std setup is done
|
13
|
-
# @raise [Karafka::Errors::
|
14
|
-
# doesn't match with
|
12
|
+
# @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
|
13
|
+
# doesn't match with the config contract
|
15
14
|
def boot!
|
15
|
+
initialize!
|
16
16
|
Setup::Config.validate!
|
17
17
|
Setup::Config.setup_components
|
18
|
-
|
18
|
+
initialized!
|
19
19
|
end
|
20
20
|
|
21
21
|
# @return [Karafka::Routing::Builder] consumers builder instance
|
22
22
|
def consumer_groups
|
23
|
-
|
23
|
+
config.internal.routing_builder
|
24
|
+
end
|
25
|
+
|
26
|
+
# Triggers reload of all cached Karafka app components, so we can use in-process
|
27
|
+
# in-development hot code reloading without Karafka process restart
|
28
|
+
def reload
|
29
|
+
Karafka::Persistence::Consumers.clear
|
30
|
+
Karafka::Persistence::Topics.clear
|
31
|
+
config.internal.routing_builder.reload
|
24
32
|
end
|
25
33
|
|
26
34
|
Status.instance_methods(false).each do |delegated|
|
27
35
|
define_method(delegated) do
|
28
|
-
|
36
|
+
App.config.internal.status.send(delegated)
|
29
37
|
end
|
30
38
|
end
|
31
39
|
|
@@ -11,18 +11,18 @@ module Karafka
|
|
11
11
|
module AttributesMap
|
12
12
|
class << self
|
13
13
|
# What settings should go where in ruby-kafka
|
14
|
+
# @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
|
14
15
|
# @note All other settings will be passed to Kafka.new method invocation.
|
15
16
|
# All elements in this hash are just edge cases
|
16
|
-
|
17
|
-
def config_adapter
|
17
|
+
def api_adapter
|
18
18
|
{
|
19
19
|
consumer: %i[
|
20
20
|
session_timeout offset_commit_interval offset_commit_threshold
|
21
|
-
offset_retention_time heartbeat_interval
|
21
|
+
offset_retention_time heartbeat_interval fetcher_max_queue_size
|
22
22
|
],
|
23
|
-
|
24
|
-
|
25
|
-
|
23
|
+
subscribe: %i[start_from_beginning max_bytes_per_partition],
|
24
|
+
consumption: %i[min_bytes max_bytes max_wait_time],
|
25
|
+
pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
|
26
26
|
# All the options that are under kafka config namespace, but are not used
|
27
27
|
# directly with kafka api, but from the Karafka user perspective, they are
|
28
28
|
# still related to kafka. They should not be proxied anywhere
|
@@ -32,13 +32,12 @@ module Karafka
|
|
32
32
|
|
33
33
|
# @return [Array<Symbol>] properties that can be set on a per topic level
|
34
34
|
def topic
|
35
|
-
(
|
35
|
+
(api_adapter[:subscribe] + %i[
|
36
36
|
backend
|
37
37
|
name
|
38
|
-
|
38
|
+
deserializer
|
39
39
|
responder
|
40
40
|
batch_consuming
|
41
|
-
persistent
|
42
41
|
]).uniq
|
43
42
|
end
|
44
43
|
|
@@ -48,17 +47,19 @@ module Karafka
|
|
48
47
|
# Thanks to this solution, if any new setting is available for ruby-kafka, we just need
|
49
48
|
# to add it to our configuration class and it will be handled automatically.
|
50
49
|
def consumer_group
|
51
|
-
# @note We don't ignore the
|
50
|
+
# @note We don't ignore the api_adapter[:ignored] values as they should be ignored
|
52
51
|
# only when proxying details go ruby-kafka. We use ignored fields internally in karafka
|
53
|
-
ignored_settings =
|
54
|
-
defined_settings =
|
52
|
+
ignored_settings = api_adapter[:subscribe]
|
53
|
+
defined_settings = api_adapter.values.flatten
|
55
54
|
karafka_settings = %i[batch_fetching]
|
56
|
-
# This is a
|
55
|
+
# This is a dirty and bad hack of dry-configurable to get keys before setting values
|
57
56
|
dynamically_proxied = Karafka::Setup::Config
|
58
57
|
._settings
|
58
|
+
.settings
|
59
59
|
.find { |s| s.name == :kafka }
|
60
60
|
.value
|
61
|
-
.
|
61
|
+
.names
|
62
|
+
.to_a
|
62
63
|
|
63
64
|
(defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
|
64
65
|
end
|
@@ -4,41 +4,33 @@
|
|
4
4
|
module Karafka
|
5
5
|
# Base consumer from which all Karafka consumers should inherit
|
6
6
|
class BaseConsumer
|
7
|
-
extend ActiveSupport::DescendantsTracker
|
8
7
|
extend Forwardable
|
9
8
|
|
10
9
|
# Allows us to mark messages as consumed for non-automatic mode without having
|
11
10
|
# to use consumer client directly. We do this that way, because most of the people should not
|
12
11
|
# mess with the client instance directly (just in case)
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
# @param topic [Karafka::Routing::Topic]
|
23
|
-
# @return [Karafka::Routing::Topic] assigned topic
|
24
|
-
def topic=(topic)
|
25
|
-
@topic = topic
|
26
|
-
Consumers::Includer.call(self)
|
27
|
-
end
|
12
|
+
%i[
|
13
|
+
mark_as_consumed
|
14
|
+
mark_as_consumed!
|
15
|
+
trigger_heartbeat
|
16
|
+
trigger_heartbeat!
|
17
|
+
].each do |delegated_method_name|
|
18
|
+
def_delegator :client, delegated_method_name
|
19
|
+
|
20
|
+
private delegated_method_name
|
28
21
|
end
|
29
22
|
|
30
23
|
# @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
#
|
36
|
-
#
|
37
|
-
# @param
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
@params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
|
24
|
+
attr_reader :topic
|
25
|
+
# @return [Karafka::Params:ParamsBatch] current params batch
|
26
|
+
attr_accessor :params_batch
|
27
|
+
|
28
|
+
# Assigns a topic to a consumer and builds up proper consumer functionalities
|
29
|
+
# so that it can cooperate with the topic settings
|
30
|
+
# @param topic [Karafka::Routing::Topic]
|
31
|
+
def initialize(topic)
|
32
|
+
@topic = topic
|
33
|
+
Consumers::Includer.call(self)
|
42
34
|
end
|
43
35
|
|
44
36
|
# Executes the default consumer flow.
|
@@ -48,9 +40,6 @@ module Karafka
|
|
48
40
|
|
49
41
|
private
|
50
42
|
|
51
|
-
# We make it private as it should be accessible only from the inside of a consumer
|
52
|
-
attr_reader :params_batch
|
53
|
-
|
54
43
|
# @return [Karafka::Connection::Client] messages consuming client that can be used to
|
55
44
|
# commit manually offset or pause / stop consumer based on the business logic
|
56
45
|
def client
|
@@ -39,7 +39,7 @@ module Karafka
|
|
39
39
|
#
|
40
40
|
# @example Multiple times used topic
|
41
41
|
# class Responder < BaseResponder
|
42
|
-
# topic :required_topic
|
42
|
+
# topic :required_topic
|
43
43
|
#
|
44
44
|
# def respond(data)
|
45
45
|
# data.each do |subset|
|
@@ -48,6 +48,17 @@ module Karafka
|
|
48
48
|
# end
|
49
49
|
# end
|
50
50
|
#
|
51
|
+
# @example Specify serializer for a topic
|
52
|
+
# class Responder < BaseResponder
|
53
|
+
# topic :xml_topic, serializer: MyXMLSerializer
|
54
|
+
#
|
55
|
+
# def respond(data)
|
56
|
+
# data.each do |subset|
|
57
|
+
# respond_to :xml_topic, subset
|
58
|
+
# end
|
59
|
+
# end
|
60
|
+
# end
|
61
|
+
#
|
51
62
|
# @example Accept multiple arguments to a respond method
|
52
63
|
# class Responder < BaseResponder
|
53
64
|
# topic :users_actions
|
@@ -59,31 +70,35 @@ module Karafka
|
|
59
70
|
# end
|
60
71
|
# end
|
61
72
|
class BaseResponder
|
62
|
-
#
|
63
|
-
|
64
|
-
|
65
|
-
# Schema that we can use to control and/or require some additional details upon options
|
66
|
-
# that are being passed to the producer. This can be in particular useful if we want to make
|
67
|
-
# sure that for example partition_key is always present.
|
68
|
-
class_attribute :options_schema
|
73
|
+
# Responder usage contract
|
74
|
+
CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
|
69
75
|
|
70
|
-
|
76
|
+
private_constant :CONTRACT
|
71
77
|
|
72
78
|
class << self
|
79
|
+
# Definitions of all topics that we want to be able to use in this responder should go here
|
80
|
+
attr_accessor :topics
|
81
|
+
# Contract that we can use to control and/or require some additional details upon options
|
82
|
+
# that are being passed to the producer. This can be in particular useful if we want to make
|
83
|
+
# sure that for example partition_key is always present.
|
84
|
+
attr_accessor :options_contract
|
85
|
+
|
73
86
|
# Registers a topic as on to which we will be able to respond
|
74
87
|
# @param topic_name [Symbol, String] name of topic to which we want to respond
|
75
88
|
# @param options [Hash] hash with optional configuration details
|
76
89
|
def topic(topic_name, options = {})
|
90
|
+
options[:serializer] ||= Karafka::App.config.serializer
|
91
|
+
options[:registered] = true
|
77
92
|
self.topics ||= {}
|
78
|
-
topic_obj = Responders::Topic.new(topic_name, options
|
93
|
+
topic_obj = Responders::Topic.new(topic_name, options)
|
79
94
|
self.topics[topic_obj.name] = topic_obj
|
80
95
|
end
|
81
96
|
|
82
97
|
# A simple alias for easier standalone responder usage.
|
83
|
-
# Instead of building it with new.call it allows (in case of
|
98
|
+
# Instead of building it with new.call it allows (in case of using JSON serializer)
|
84
99
|
# to just run it directly from the class level
|
85
100
|
# @param data Anything that we want to respond with
|
86
|
-
# @example Send user data with a responder
|
101
|
+
# @example Send user data with a responder
|
87
102
|
# UsersCreatedResponder.call(@created_user)
|
88
103
|
def call(*data)
|
89
104
|
# Just in case there were no topics defined for a responder, we initialize with
|
@@ -93,12 +108,11 @@ module Karafka
|
|
93
108
|
end
|
94
109
|
end
|
95
110
|
|
111
|
+
attr_reader :messages_buffer
|
112
|
+
|
96
113
|
# Creates a responder object
|
97
|
-
# @param parser_class [Class] parser class that we can use to generate appropriate string
|
98
|
-
# or nothing if we want to default to Karafka::Parsers::Json
|
99
114
|
# @return [Karafka::BaseResponder] base responder descendant responder
|
100
|
-
def initialize
|
101
|
-
@parser_class = parser_class
|
115
|
+
def initialize
|
102
116
|
@messages_buffer = {}
|
103
117
|
end
|
104
118
|
|
@@ -107,7 +121,7 @@ module Karafka
|
|
107
121
|
# @note We know that validators should be executed also before sending data to topics, however
|
108
122
|
# the implementation gets way more complicated then, that's why we check after everything
|
109
123
|
# was sent using responder
|
110
|
-
# @example Send user data with a responder
|
124
|
+
# @example Send user data with a responder
|
111
125
|
# UsersCreatedResponder.new.call(@created_user)
|
112
126
|
# @example Send user data with a responder using non default Parser
|
113
127
|
# UsersCreatedResponder.new(MyParser).call(@created_user)
|
@@ -134,25 +148,26 @@ module Karafka
|
|
134
148
|
topic.to_h.merge!(usage_count: usage.count)
|
135
149
|
end
|
136
150
|
|
137
|
-
result =
|
151
|
+
result = CONTRACT.call(
|
138
152
|
registered_topics: registered_topics,
|
139
153
|
used_topics: used_topics
|
140
154
|
)
|
141
155
|
|
142
156
|
return if result.success?
|
143
157
|
|
144
|
-
raise Karafka::Errors::
|
158
|
+
raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
|
145
159
|
end
|
146
160
|
|
147
161
|
# Checks if we met all the options requirements before sending them to the producer.
|
148
162
|
def validate_options!
|
149
|
-
return true unless self.class.
|
163
|
+
return true unless self.class.options_contract
|
150
164
|
|
151
165
|
messages_buffer.each_value do |messages_set|
|
152
166
|
messages_set.each do |message_data|
|
153
|
-
result = self.class.
|
167
|
+
result = self.class.options_contract.call(message_data.last)
|
154
168
|
next if result.success?
|
155
|
-
|
169
|
+
|
170
|
+
raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
|
156
171
|
end
|
157
172
|
end
|
158
173
|
end
|
@@ -163,13 +178,18 @@ module Karafka
|
|
163
178
|
def deliver!
|
164
179
|
messages_buffer.each_value do |data_elements|
|
165
180
|
data_elements.each do |data, options|
|
166
|
-
|
181
|
+
# We map this topic name, so it will match namespaced/etc topic in Kafka
|
182
|
+
# @note By default will not change topic (if default mapper used)
|
183
|
+
mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
|
184
|
+
external_options = options.merge(topic: mapped_topic)
|
185
|
+
producer(options).call(data, external_options)
|
167
186
|
end
|
168
187
|
end
|
169
188
|
end
|
170
189
|
|
171
190
|
# Method that needs to be implemented in a subclass. It should handle responding
|
172
191
|
# on registered topics
|
192
|
+
# @param _data [Object] anything that we want to use to send to Kafka
|
173
193
|
# @raise [NotImplementedError] This method needs to be implemented in a subclass
|
174
194
|
def respond(*_data)
|
175
195
|
raise NotImplementedError, 'Implement this in a subclass'
|
@@ -179,7 +199,7 @@ module Karafka
|
|
179
199
|
# as many times as we need. Especially when we have 1:n flow
|
180
200
|
# @param topic [Symbol, String] topic to which we want to respond
|
181
201
|
# @param data [String, Object] string or object that we want to send
|
182
|
-
# @param options [Hash] options for waterdrop (e.g. partition_key)
|
202
|
+
# @param options [Hash] options for waterdrop (e.g. partition_key).
|
183
203
|
# @note Respond to does not accept multiple data arguments.
|
184
204
|
def respond_to(topic, data, options = {})
|
185
205
|
# We normalize the format to string, as WaterDrop and Ruby-Kafka support only
|
@@ -188,17 +208,19 @@ module Karafka
|
|
188
208
|
|
189
209
|
messages_buffer[topic] ||= []
|
190
210
|
messages_buffer[topic] << [
|
191
|
-
|
192
|
-
|
193
|
-
# @note By default will not change topic (if default mapper used)
|
194
|
-
options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
|
211
|
+
self.class.topics[topic].serializer.call(data),
|
212
|
+
options.merge(topic: topic)
|
195
213
|
]
|
196
214
|
end
|
197
215
|
|
198
216
|
# @param options [Hash] options for waterdrop
|
199
217
|
# @return [Class] WaterDrop producer (sync or async based on the settings)
|
200
218
|
def producer(options)
|
201
|
-
options[:async
|
219
|
+
if self.class.topics[options[:topic]].async?
|
220
|
+
WaterDrop::AsyncProducer
|
221
|
+
else
|
222
|
+
WaterDrop::SyncProducer
|
223
|
+
end
|
202
224
|
end
|
203
225
|
end
|
204
226
|
end
|