karafka 1.2.13 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +2 -0
- data/{.coditsu.yml → .coditsu/ci.yml} +1 -1
- data/.console_irbrc +1 -3
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +8 -21
- data/CHANGELOG.md +91 -18
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +2 -5
- data/Gemfile.lock +79 -65
- data/README.md +10 -11
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/karafka.gemspec +14 -12
- data/lib/karafka.rb +8 -15
- data/lib/karafka/app.rb +14 -6
- data/lib/karafka/attributes_map.rb +5 -12
- data/lib/karafka/base_consumer.rb +19 -30
- data/lib/karafka/base_responder.rb +45 -27
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +5 -2
- data/lib/karafka/cli/info.rb +3 -1
- data/lib/karafka/cli/install.rb +30 -6
- data/lib/karafka/cli/server.rb +11 -6
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/api_adapter.rb +11 -4
- data/lib/karafka/connection/batch_delegator.rb +51 -0
- data/lib/karafka/connection/builder.rb +1 -1
- data/lib/karafka/connection/client.rb +30 -20
- data/lib/karafka/connection/listener.rb +24 -13
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/callbacks.rb +32 -15
- data/lib/karafka/consumers/includer.rb +30 -18
- data/lib/karafka/consumers/metadata.rb +10 -0
- data/lib/karafka/consumers/responders.rb +2 -2
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +31 -0
- data/lib/karafka/errors.rb +17 -16
- data/lib/karafka/fetcher.rb +28 -30
- data/lib/karafka/helpers/class_matcher.rb +6 -2
- data/lib/karafka/helpers/config_retriever.rb +1 -1
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +9 -6
- data/lib/karafka/instrumentation/monitor.rb +15 -9
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
- data/lib/karafka/params/builders/metadata.rb +33 -0
- data/lib/karafka/params/builders/params.rb +36 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +35 -0
- data/lib/karafka/params/params.rb +68 -0
- data/lib/karafka/params/params_batch.rb +35 -20
- data/lib/karafka/patches/ruby_kafka.rb +21 -8
- data/lib/karafka/persistence/client.rb +15 -11
- data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +0 -2
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +36 -8
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +9 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/topic.rb +5 -3
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +25 -27
- data/lib/karafka/setup/config.rb +63 -37
- data/lib/karafka/setup/configurators/water_drop.rb +7 -3
- data/lib/karafka/setup/dsl.rb +0 -1
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +90 -57
- metadata.gz.sig +0 -0
- data/lib/karafka/callbacks.rb +0 -30
- data/lib/karafka/callbacks/config.rb +0 -22
- data/lib/karafka/callbacks/dsl.rb +0 -16
- data/lib/karafka/connection/delegator.rb +0 -46
- data/lib/karafka/instrumentation/listener.rb +0 -112
- data/lib/karafka/loader.rb +0 -28
- data/lib/karafka/params/dsl.rb +0 -158
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -33
- data/lib/karafka/persistence/topic.rb +0 -29
- data/lib/karafka/schemas/config.rb +0 -24
- data/lib/karafka/schemas/consumer_group.rb +0 -79
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -29
- data/lib/karafka/setup/configurators/params.rb +0 -25
- data/lib/karafka/templates/karafka.rb.example +0 -54
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Karafka
|
|
4
|
-
module Schemas
|
|
5
|
-
# Schema for validating correctness of the server cli command options
|
|
6
|
-
# We validate some basics + the list of consumer_groups on which we want to use, to make
|
|
7
|
-
# sure that all of them are defined, plus that a pidfile does not exist
|
|
8
|
-
ServerCliOptions = Dry::Validation.Schema do
|
|
9
|
-
configure do
|
|
10
|
-
option :consumer_groups
|
|
11
|
-
|
|
12
|
-
def self.messages
|
|
13
|
-
super.merge(
|
|
14
|
-
en: {
|
|
15
|
-
errors: {
|
|
16
|
-
consumer_groups_inclusion: 'Unknown consumer group.',
|
|
17
|
-
pid_existence: 'Pidfile already exists.'
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
)
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
optional(:pid).filled(:str?)
|
|
25
|
-
optional(:daemon).filled(:bool?)
|
|
26
|
-
optional(:consumer_groups).filled(:array?)
|
|
27
|
-
|
|
28
|
-
validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
|
|
29
|
-
# If there were no consumer_groups declared in the server cli, it means that we will
|
|
30
|
-
# run all of them and no need to validate them here at all
|
|
31
|
-
if consumer_groups.nil?
|
|
32
|
-
true
|
|
33
|
-
else
|
|
34
|
-
(consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
validate(pid_existence: :pid) do |pid|
|
|
39
|
-
pid ? !File.exist?(pid) : true
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
end
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Karafka
|
|
4
|
-
module Setup
|
|
5
|
-
# Configurators module is used to enclose all the external dependencies configurations
|
|
6
|
-
# upon which Karafka depents
|
|
7
|
-
class Configurators
|
|
8
|
-
# Karafka has some components that it relies on (like Sidekiq)
|
|
9
|
-
# We need to configure all of them only when the framework was set up.
|
|
10
|
-
# Any class that descends from this one will be automatically invoked upon setup (after it)
|
|
11
|
-
# @note This should be used only for internal Karafka dependencies configuration
|
|
12
|
-
# End users configuration should go to the after_init block
|
|
13
|
-
# @example Configure an Example class
|
|
14
|
-
# class ExampleConfigurator < Base
|
|
15
|
-
# def setup
|
|
16
|
-
# ExampleClass.logger = Karafka.logger
|
|
17
|
-
# ExampleClass.redis = config.redis
|
|
18
|
-
# end
|
|
19
|
-
# end
|
|
20
|
-
class Base
|
|
21
|
-
# @param _config [Karafka::Config] config instance
|
|
22
|
-
# This method needs to be implemented in a subclass
|
|
23
|
-
def self.setup(_config)
|
|
24
|
-
raise NotImplementedError
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Karafka
|
|
4
|
-
module Setup
|
|
5
|
-
class Configurators
|
|
6
|
-
# Karafka::Params::Params are dynamically built based on user defined parent class
|
|
7
|
-
# so we cannot just require it, we need to initialize it after user is done with
|
|
8
|
-
# the framework configuration. This is a configurator that does exactly that.
|
|
9
|
-
class Params < Base
|
|
10
|
-
# Builds up Karafka::Params::Params class with user defined parent class
|
|
11
|
-
# @param config [Karafka::Setup::Config] Config we can user to setup things
|
|
12
|
-
def self.setup(config)
|
|
13
|
-
return if defined? Karafka::Params::Params
|
|
14
|
-
|
|
15
|
-
Karafka::Params.const_set(
|
|
16
|
-
'Params',
|
|
17
|
-
Class
|
|
18
|
-
.new(config.params_base_class)
|
|
19
|
-
.tap { |klass| klass.include(Karafka::Params::Dsl) }
|
|
20
|
-
)
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
end
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# Non Ruby on Rails setup
|
|
4
|
-
ENV['RACK_ENV'] ||= 'development'
|
|
5
|
-
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
|
6
|
-
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
|
7
|
-
Karafka::Loader.load(Karafka::App.root)
|
|
8
|
-
|
|
9
|
-
# Ruby on Rails setup
|
|
10
|
-
# Remove whole non-Rails setup that is above and uncomment the 4 lines below
|
|
11
|
-
# ENV['RAILS_ENV'] ||= 'development'
|
|
12
|
-
# ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
|
|
13
|
-
# require ::File.expand_path('../config/environment', __FILE__)
|
|
14
|
-
# Rails.application.eager_load!
|
|
15
|
-
|
|
16
|
-
class KarafkaApp < Karafka::App
|
|
17
|
-
setup do |config|
|
|
18
|
-
config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
|
|
19
|
-
config.client_id = 'example_app'
|
|
20
|
-
config.backend = :inline
|
|
21
|
-
config.batch_fetching = true
|
|
22
|
-
# Uncomment this for Rails app integration
|
|
23
|
-
# config.logger = Rails.logger
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
after_init do |config|
|
|
27
|
-
# Put here all the things you want to do after the Karafka framework
|
|
28
|
-
# initialization
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# Comment out this part if you are not using instrumentation and/or you are not
|
|
32
|
-
# interested in logging events for certain environments. Since instrumentation
|
|
33
|
-
# notifications add extra boilerplate, if you want to achieve max performance,
|
|
34
|
-
# listen to only what you really need for given environment.
|
|
35
|
-
Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
|
|
36
|
-
|
|
37
|
-
consumer_groups.draw do
|
|
38
|
-
# topic :example do
|
|
39
|
-
# consumer ExampleConsumer
|
|
40
|
-
# end
|
|
41
|
-
|
|
42
|
-
# consumer_group :bigger_group do
|
|
43
|
-
# topic :test do
|
|
44
|
-
# consumer TestConsumer
|
|
45
|
-
# end
|
|
46
|
-
#
|
|
47
|
-
# topic :test2 do
|
|
48
|
-
# consumer Test2Consumer
|
|
49
|
-
# end
|
|
50
|
-
# end
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
KarafkaApp.boot!
|