karafka 1.2.10 → 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/{.coditsu.yml → .coditsu/ci.yml} +1 -1
- data/.console_irbrc +1 -3
- data/.github/FUNDING.yml +3 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +4 -17
- data/CHANGELOG.md +73 -0
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +2 -5
- data/Gemfile.lock +77 -66
- data/README.md +10 -11
- data/bin/karafka +1 -1
- data/certs/mensfeld.pem +25 -0
- data/config/errors.yml +38 -5
- data/karafka.gemspec +16 -14
- data/lib/karafka.rb +7 -15
- data/lib/karafka/app.rb +14 -6
- data/lib/karafka/attributes_map.rb +7 -6
- data/lib/karafka/base_consumer.rb +19 -30
- data/lib/karafka/base_responder.rb +45 -27
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/console.rb +11 -9
- data/lib/karafka/cli/flow.rb +0 -1
- data/lib/karafka/cli/info.rb +3 -1
- data/lib/karafka/cli/install.rb +30 -6
- data/lib/karafka/cli/server.rb +11 -6
- data/lib/karafka/code_reloader.rb +67 -0
- data/lib/karafka/connection/api_adapter.rb +11 -4
- data/lib/karafka/connection/batch_delegator.rb +51 -0
- data/lib/karafka/connection/builder.rb +1 -1
- data/lib/karafka/connection/client.rb +30 -20
- data/lib/karafka/connection/listener.rb +24 -13
- data/lib/karafka/connection/message_delegator.rb +36 -0
- data/lib/karafka/consumers/callbacks.rb +32 -15
- data/lib/karafka/consumers/includer.rb +30 -18
- data/lib/karafka/consumers/metadata.rb +10 -0
- data/lib/karafka/consumers/responders.rb +2 -2
- data/lib/karafka/contracts.rb +10 -0
- data/lib/karafka/contracts/config.rb +21 -0
- data/lib/karafka/contracts/consumer_group.rb +206 -0
- data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
- data/lib/karafka/contracts/responder_usage.rb +54 -0
- data/lib/karafka/contracts/server_cli_options.rb +29 -0
- data/lib/karafka/errors.rb +17 -16
- data/lib/karafka/fetcher.rb +28 -30
- data/lib/karafka/helpers/class_matcher.rb +11 -1
- data/lib/karafka/helpers/config_retriever.rb +1 -1
- data/lib/karafka/helpers/inflector.rb +26 -0
- data/lib/karafka/helpers/multi_delegator.rb +0 -1
- data/lib/karafka/instrumentation/logger.rb +9 -6
- data/lib/karafka/instrumentation/monitor.rb +15 -9
- data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
- data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
- data/lib/karafka/params/builders/metadata.rb +33 -0
- data/lib/karafka/params/builders/params.rb +36 -0
- data/lib/karafka/params/builders/params_batch.rb +25 -0
- data/lib/karafka/params/metadata.rb +35 -0
- data/lib/karafka/params/params.rb +68 -0
- data/lib/karafka/params/params_batch.rb +35 -20
- data/lib/karafka/patches/ruby_kafka.rb +21 -8
- data/lib/karafka/persistence/client.rb +15 -11
- data/lib/karafka/persistence/{consumer.rb → consumers.rb} +20 -13
- data/lib/karafka/persistence/topics.rb +48 -0
- data/lib/karafka/process.rb +0 -2
- data/lib/karafka/responders/builder.rb +1 -1
- data/lib/karafka/responders/topic.rb +6 -8
- data/lib/karafka/routing/builder.rb +36 -8
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/consumer_mapper.rb +9 -9
- data/lib/karafka/routing/proxy.rb +10 -1
- data/lib/karafka/routing/topic.rb +5 -3
- data/lib/karafka/routing/topic_mapper.rb +16 -18
- data/lib/karafka/serialization/json/deserializer.rb +27 -0
- data/lib/karafka/serialization/json/serializer.rb +31 -0
- data/lib/karafka/server.rb +25 -27
- data/lib/karafka/setup/config.rb +65 -37
- data/lib/karafka/setup/configurators/water_drop.rb +7 -3
- data/lib/karafka/setup/dsl.rb +0 -1
- data/lib/karafka/status.rb +7 -3
- data/lib/karafka/templates/{application_consumer.rb.example → application_consumer.rb.erb} +2 -1
- data/lib/karafka/templates/{application_responder.rb.example → application_responder.rb.erb} +0 -0
- data/lib/karafka/templates/karafka.rb.erb +92 -0
- data/lib/karafka/version.rb +1 -1
- metadata +94 -61
- metadata.gz.sig +0 -0
- data/lib/karafka/callbacks.rb +0 -30
- data/lib/karafka/callbacks/config.rb +0 -22
- data/lib/karafka/callbacks/dsl.rb +0 -16
- data/lib/karafka/connection/delegator.rb +0 -46
- data/lib/karafka/instrumentation/listener.rb +0 -112
- data/lib/karafka/loader.rb +0 -28
- data/lib/karafka/params/dsl.rb +0 -158
- data/lib/karafka/parsers/json.rb +0 -38
- data/lib/karafka/patches/dry_configurable.rb +0 -35
- data/lib/karafka/persistence/topic.rb +0 -29
- data/lib/karafka/schemas/config.rb +0 -24
- data/lib/karafka/schemas/consumer_group.rb +0 -78
- data/lib/karafka/schemas/consumer_group_topic.rb +0 -18
- data/lib/karafka/schemas/responder_usage.rb +0 -39
- data/lib/karafka/schemas/server_cli_options.rb +0 -43
- data/lib/karafka/setup/configurators/base.rb +0 -29
- data/lib/karafka/setup/configurators/params.rb +0 -25
- data/lib/karafka/templates/karafka.rb.example +0 -54
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Schemas
|
5
|
-
# Consumer group topic validation rules
|
6
|
-
ConsumerGroupTopic = Dry::Validation.Schema do
|
7
|
-
required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
8
|
-
required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
9
|
-
required(:backend).filled(included_in?: %i[inline sidekiq])
|
10
|
-
required(:consumer).filled
|
11
|
-
required(:parser).filled
|
12
|
-
required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
|
13
|
-
required(:start_from_beginning).filled(:bool?)
|
14
|
-
required(:batch_consuming).filled(:bool?)
|
15
|
-
required(:persistent).filled(:bool?)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Schemas
|
5
|
-
# Validator to check responder topic usage
|
6
|
-
ResponderUsageTopic = Dry::Validation.Schema do
|
7
|
-
required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
|
8
|
-
required(:required).filled(:bool?)
|
9
|
-
required(:multiple_usage).filled(:bool?)
|
10
|
-
required(:usage_count).filled(:int?, gteq?: 0)
|
11
|
-
required(:registered).filled(eql?: true)
|
12
|
-
required(:async).filled(:bool?)
|
13
|
-
|
14
|
-
rule(
|
15
|
-
required_usage: %i[required usage_count]
|
16
|
-
) do |required, usage_count|
|
17
|
-
required.true? > usage_count.gteq?(1)
|
18
|
-
end
|
19
|
-
|
20
|
-
rule(
|
21
|
-
multiple_usage_permission: %i[multiple_usage usage_count]
|
22
|
-
) do |multiple_usage, usage_count|
|
23
|
-
usage_count.gt?(1) > multiple_usage.true?
|
24
|
-
end
|
25
|
-
|
26
|
-
rule(
|
27
|
-
multiple_usage_block: %i[multiple_usage usage_count]
|
28
|
-
) do |multiple_usage, usage_count|
|
29
|
-
multiple_usage.false? > usage_count.lteq?(1)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
# Validator to check that everything in a responder flow matches responder rules
|
34
|
-
ResponderUsage = Dry::Validation.Schema do
|
35
|
-
required(:used_topics) { filled? > each { schema(ResponderUsageTopic) } }
|
36
|
-
required(:registered_topics) { filled? > each { schema(ResponderUsageTopic) } }
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Schemas
|
5
|
-
# Schema for validating correctness of the server cli command options
|
6
|
-
# We validate some basics + the list of consumer_groups on which we want to use, to make
|
7
|
-
# sure that all of them are defined, plus that a pidfile does not exist
|
8
|
-
ServerCliOptions = Dry::Validation.Schema do
|
9
|
-
configure do
|
10
|
-
option :consumer_groups
|
11
|
-
|
12
|
-
def self.messages
|
13
|
-
super.merge(
|
14
|
-
en: {
|
15
|
-
errors: {
|
16
|
-
consumer_groups_inclusion: 'Unknown consumer group.',
|
17
|
-
pid_existence: 'Pidfile already exists.'
|
18
|
-
}
|
19
|
-
}
|
20
|
-
)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
optional(:pid).filled(:str?)
|
25
|
-
optional(:daemon).filled(:bool?)
|
26
|
-
optional(:consumer_groups).filled(:array?)
|
27
|
-
|
28
|
-
validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
|
29
|
-
# If there were no consumer_groups declared in the server cli, it means that we will
|
30
|
-
# run all of them and no need to validate them here at all
|
31
|
-
if consumer_groups.nil?
|
32
|
-
true
|
33
|
-
else
|
34
|
-
(consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
validate(pid_existence: :pid) do |pid|
|
39
|
-
pid ? !File.exist?(pid) : true
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,29 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Setup
|
5
|
-
# Configurators module is used to enclose all the external dependencies configurations
|
6
|
-
# upon which Karafka depents
|
7
|
-
class Configurators
|
8
|
-
# Karafka has some components that it relies on (like Sidekiq)
|
9
|
-
# We need to configure all of them only when the framework was set up.
|
10
|
-
# Any class that descends from this one will be automatically invoked upon setup (after it)
|
11
|
-
# @note This should be used only for internal Karafka dependencies configuration
|
12
|
-
# End users configuration should go to the after_init block
|
13
|
-
# @example Configure an Example class
|
14
|
-
# class ExampleConfigurator < Base
|
15
|
-
# def setup
|
16
|
-
# ExampleClass.logger = Karafka.logger
|
17
|
-
# ExampleClass.redis = config.redis
|
18
|
-
# end
|
19
|
-
# end
|
20
|
-
class Base
|
21
|
-
# @param _config [Karafka::Config] config instance
|
22
|
-
# This method needs to be implemented in a subclass
|
23
|
-
def self.setup(_config)
|
24
|
-
raise NotImplementedError
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Setup
|
5
|
-
class Configurators
|
6
|
-
# Karafka::Params::Params are dynamically built based on user defined parent class
|
7
|
-
# so we cannot just require it, we need to initialize it after user is done with
|
8
|
-
# the framework configuration. This is a configurator that does exactly that.
|
9
|
-
class Params < Base
|
10
|
-
# Builds up Karafka::Params::Params class with user defined parent class
|
11
|
-
# @param config [Karafka::Setup::Config] Config we can user to setup things
|
12
|
-
def self.setup(config)
|
13
|
-
return if defined? Karafka::Params::Params
|
14
|
-
|
15
|
-
Karafka::Params.const_set(
|
16
|
-
'Params',
|
17
|
-
Class
|
18
|
-
.new(config.params_base_class)
|
19
|
-
.tap { |klass| klass.include(Karafka::Params::Dsl) }
|
20
|
-
)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Non Ruby on Rails setup
|
4
|
-
ENV['RACK_ENV'] ||= 'development'
|
5
|
-
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
6
|
-
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
7
|
-
Karafka::Loader.load(Karafka::App.root)
|
8
|
-
|
9
|
-
# Ruby on Rails setup
|
10
|
-
# Remove whole non-Rails setup that is above and uncomment the 4 lines below
|
11
|
-
# ENV['RAILS_ENV'] ||= 'development'
|
12
|
-
# ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
|
13
|
-
# require ::File.expand_path('../config/environment', __FILE__)
|
14
|
-
# Rails.application.eager_load!
|
15
|
-
|
16
|
-
class KarafkaApp < Karafka::App
|
17
|
-
setup do |config|
|
18
|
-
config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
|
19
|
-
config.client_id = 'example_app'
|
20
|
-
config.backend = :inline
|
21
|
-
config.batch_fetching = true
|
22
|
-
# Uncomment this for Rails app integration
|
23
|
-
# config.logger = Rails.logger
|
24
|
-
end
|
25
|
-
|
26
|
-
after_init do |config|
|
27
|
-
# Put here all the things you want to do after the Karafka framework
|
28
|
-
# initialization
|
29
|
-
end
|
30
|
-
|
31
|
-
# Comment out this part if you are not using instrumentation and/or you are not
|
32
|
-
# interested in logging events for certain environments. Since instrumentation
|
33
|
-
# notifications add extra boilerplate, if you want to achieve max performance,
|
34
|
-
# listen to only what you really need for given environment.
|
35
|
-
Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
|
36
|
-
|
37
|
-
consumer_groups.draw do
|
38
|
-
# topic :example do
|
39
|
-
# consumer ExampleConsumer
|
40
|
-
# end
|
41
|
-
|
42
|
-
# consumer_group :bigger_group do
|
43
|
-
# topic :test do
|
44
|
-
# consumer TestConsumer
|
45
|
-
# end
|
46
|
-
#
|
47
|
-
# topic :test2 do
|
48
|
-
# consumer Test2Consumer
|
49
|
-
# end
|
50
|
-
# end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
KarafkaApp.boot!
|