karafka 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +40 -16
- data/.rspec +4 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +459 -796
- data/Gemfile +3 -2
- data/Gemfile.lock +44 -32
- data/LICENSE +1 -1
- data/README.md +22 -17
- data/bin/integrations +86 -26
- data/bin/karafka +1 -13
- data/bin/rspecs +6 -0
- data/bin/verify_license_integrity +37 -0
- data/certs/cert_chain.pem +26 -0
- data/config/{errors.yml → locales/errors.yml} +34 -6
- data/config/locales/pro_errors.yml +34 -0
- data/docker-compose.yml +4 -30
- data/karafka.gemspec +17 -10
- data/lib/active_job/karafka.rb +3 -7
- data/lib/active_job/queue_adapters/karafka_adapter.rb +7 -1
- data/lib/karafka/active_job/consumer.rb +29 -6
- data/lib/karafka/active_job/current_attributes/loading.rb +36 -0
- data/lib/karafka/active_job/current_attributes/persistence.rb +28 -0
- data/lib/karafka/active_job/current_attributes.rb +42 -0
- data/lib/karafka/active_job/dispatcher.rb +34 -3
- data/lib/karafka/active_job/job_options_contract.rb +13 -2
- data/lib/karafka/admin.rb +246 -0
- data/lib/karafka/app.rb +30 -7
- data/lib/karafka/base_consumer.rb +124 -82
- data/lib/karafka/cli/base.rb +20 -0
- data/lib/karafka/cli/console.rb +13 -8
- data/lib/karafka/cli/info.rb +3 -2
- data/lib/karafka/cli/server.rb +55 -8
- data/lib/karafka/cli/topics.rb +146 -0
- data/lib/karafka/connection/client.rb +118 -61
- data/lib/karafka/connection/consumer_group_coordinator.rb +48 -0
- data/lib/karafka/connection/listener.rb +78 -30
- data/lib/karafka/connection/listeners_batch.rb +18 -2
- data/lib/karafka/connection/rebalance_manager.rb +3 -0
- data/lib/karafka/contracts/config.rb +1 -1
- data/lib/karafka/contracts/consumer_group.rb +22 -9
- data/lib/karafka/contracts/server_cli_options.rb +71 -11
- data/lib/karafka/contracts/topic.rb +65 -0
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/embedded.rb +36 -0
- data/lib/karafka/errors.rb +6 -4
- data/lib/karafka/helpers/colorize.rb +6 -0
- data/lib/karafka/instrumentation/callbacks/error.rb +3 -4
- data/lib/karafka/instrumentation/callbacks/statistics.rb +14 -4
- data/lib/karafka/instrumentation/logger_listener.rb +151 -22
- data/lib/karafka/instrumentation/notifications.rb +26 -7
- data/lib/karafka/instrumentation/proctitle_listener.rb +7 -16
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -1
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +153 -0
- data/lib/karafka/instrumentation/vendors/datadog/{listener.rb → metrics_listener.rb} +62 -35
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +166 -0
- data/lib/karafka/licenser.rb +53 -50
- data/lib/karafka/messages/builders/batch_metadata.rb +8 -8
- data/lib/karafka/messages/builders/messages.rb +6 -2
- data/lib/karafka/messages/message.rb +14 -2
- data/lib/karafka/messages/messages.rb +5 -0
- data/lib/karafka/messages/parser.rb +14 -0
- data/lib/karafka/patches/rdkafka/bindings.rb +132 -0
- data/lib/karafka/pro/active_job/consumer.rb +10 -9
- data/lib/karafka/pro/active_job/dispatcher.rb +28 -3
- data/lib/karafka/pro/active_job/job_options_contract.rb +16 -3
- data/lib/karafka/pro/encryption/cipher.rb +58 -0
- data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
- data/lib/karafka/pro/encryption/errors.rb +24 -0
- data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
- data/lib/karafka/pro/encryption/setup/config.rb +48 -0
- data/lib/karafka/pro/encryption.rb +47 -0
- data/lib/karafka/pro/iterator.rb +253 -0
- data/lib/karafka/pro/loader.rb +52 -26
- data/lib/karafka/pro/performance_tracker.rb +10 -6
- data/lib/karafka/pro/processing/collapser.rb +62 -0
- data/lib/karafka/pro/processing/coordinator.rb +92 -16
- data/lib/karafka/pro/processing/filters/base.rb +61 -0
- data/lib/karafka/pro/processing/filters/delayer.rb +70 -0
- data/lib/karafka/pro/processing/filters/expirer.rb +51 -0
- data/lib/karafka/pro/processing/filters/throttler.rb +84 -0
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +52 -0
- data/lib/karafka/pro/processing/filters_applier.rb +104 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +7 -5
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +22 -4
- data/lib/karafka/pro/processing/partitioner.rb +33 -24
- data/lib/karafka/pro/processing/scheduler.rb +27 -8
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +68 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +74 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +72 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +76 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +64 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +68 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +64 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +69 -0
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +38 -0
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +64 -0
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +38 -0
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +58 -0
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +37 -0
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +79 -0
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +36 -0
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +52 -0
- data/lib/karafka/pro/processing/strategies/base.rb +26 -0
- data/lib/karafka/pro/processing/strategies/default.rb +105 -0
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +131 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +61 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +75 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +71 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +43 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +41 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +69 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +41 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +40 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +63 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +63 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +36 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +39 -0
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +68 -0
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +37 -0
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +40 -0
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +104 -0
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +40 -0
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +87 -0
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +69 -0
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +67 -0
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +40 -0
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +39 -0
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +80 -0
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +38 -0
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +36 -0
- data/lib/karafka/pro/processing/strategies/mom/default.rb +46 -0
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +53 -0
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +37 -0
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +35 -0
- data/lib/karafka/pro/processing/strategies/vp/default.rb +104 -0
- data/lib/karafka/pro/{contracts/base.rb → processing/strategies.rb} +6 -5
- data/lib/karafka/pro/processing/strategy_selector.rb +84 -0
- data/lib/karafka/pro/processing/virtual_offset_manager.rb +147 -0
- data/lib/karafka/pro/routing/features/base.rb +24 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +50 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +27 -0
- data/lib/karafka/pro/routing/{builder_extensions.rb → features/delaying/config.rb} +8 -11
- data/lib/karafka/pro/routing/features/delaying/contract.rb +38 -0
- data/lib/karafka/pro/routing/features/delaying/topic.rb +59 -0
- data/lib/karafka/pro/routing/features/delaying.rb +29 -0
- data/lib/karafka/pro/routing/features/expiring/config.rb +27 -0
- data/lib/karafka/pro/routing/features/expiring/contract.rb +38 -0
- data/lib/karafka/pro/routing/features/expiring/topic.rb +59 -0
- data/lib/karafka/pro/routing/features/expiring.rb +27 -0
- data/lib/karafka/pro/routing/features/filtering/config.rb +40 -0
- data/lib/karafka/pro/routing/features/filtering/contract.rb +41 -0
- data/lib/karafka/pro/routing/features/filtering/topic.rb +51 -0
- data/lib/karafka/pro/routing/features/filtering.rb +27 -0
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +28 -0
- data/lib/karafka/pro/routing/features/long_running_job/contract.rb +37 -0
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +42 -0
- data/lib/karafka/pro/routing/features/long_running_job.rb +28 -0
- data/lib/karafka/pro/routing/features/throttling/config.rb +32 -0
- data/lib/karafka/pro/routing/features/throttling/contract.rb +41 -0
- data/lib/karafka/pro/routing/features/throttling/topic.rb +69 -0
- data/lib/karafka/pro/routing/features/throttling.rb +30 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +30 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +52 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +56 -0
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +27 -0
- data/lib/karafka/process.rb +23 -8
- data/lib/karafka/processing/coordinator.rb +102 -23
- data/lib/karafka/processing/coordinators_buffer.rb +5 -1
- data/lib/karafka/processing/executor.rb +51 -21
- data/lib/karafka/processing/executors_buffer.rb +10 -26
- data/lib/karafka/processing/jobs/base.rb +6 -2
- data/lib/karafka/processing/jobs/consume.rb +8 -5
- data/lib/karafka/processing/jobs/idle.rb +24 -0
- data/lib/karafka/processing/jobs_builder.rb +2 -3
- data/lib/karafka/processing/jobs_queue.rb +14 -8
- data/lib/karafka/processing/partitioner.rb +3 -1
- data/lib/karafka/processing/result.rb +14 -1
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +44 -0
- data/lib/karafka/processing/strategies/aj_mom.rb +21 -0
- data/lib/karafka/processing/strategies/base.rb +52 -0
- data/lib/karafka/processing/strategies/default.rb +142 -0
- data/lib/karafka/processing/strategies/dlq.rb +88 -0
- data/lib/karafka/processing/strategies/dlq_mom.rb +49 -0
- data/lib/karafka/processing/strategies/mom.rb +29 -0
- data/lib/karafka/processing/strategy_selector.rb +47 -0
- data/lib/karafka/processing/worker.rb +8 -3
- data/lib/karafka/railtie.rb +24 -12
- data/lib/karafka/routing/activity_manager.rb +84 -0
- data/lib/karafka/routing/builder.rb +20 -0
- data/lib/karafka/routing/consumer_group.rb +38 -4
- data/lib/karafka/routing/consumer_mapper.rb +0 -10
- data/lib/karafka/routing/features/active_job/builder.rb +33 -0
- data/lib/karafka/routing/features/active_job/config.rb +15 -0
- data/lib/karafka/routing/features/active_job/contract.rb +41 -0
- data/lib/karafka/routing/features/active_job/topic.rb +33 -0
- data/lib/karafka/routing/features/active_job.rb +13 -0
- data/lib/karafka/routing/features/base/expander.rb +53 -0
- data/lib/karafka/routing/features/base.rb +34 -0
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +19 -0
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +42 -0
- data/lib/karafka/routing/features/dead_letter_queue/topic.rb +41 -0
- data/lib/karafka/routing/features/dead_letter_queue.rb +16 -0
- data/lib/karafka/routing/features/declaratives/config.rb +18 -0
- data/lib/karafka/routing/features/declaratives/contract.rb +30 -0
- data/lib/karafka/routing/features/declaratives/topic.rb +44 -0
- data/lib/karafka/routing/features/declaratives.rb +14 -0
- data/lib/karafka/routing/features/manual_offset_management/config.rb +15 -0
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +24 -0
- data/lib/karafka/routing/features/manual_offset_management/topic.rb +35 -0
- data/lib/karafka/routing/features/manual_offset_management.rb +18 -0
- data/lib/karafka/routing/proxy.rb +18 -20
- data/lib/karafka/routing/router.rb +27 -2
- data/lib/karafka/routing/subscription_group.rb +43 -5
- data/lib/karafka/routing/subscription_groups_builder.rb +6 -1
- data/lib/karafka/routing/topic.rb +17 -5
- data/lib/karafka/routing/topics.rb +8 -0
- data/lib/karafka/runner.rb +1 -0
- data/lib/karafka/serialization/json/deserializer.rb +1 -0
- data/lib/karafka/server.rb +58 -28
- data/lib/karafka/setup/attributes_map.rb +344 -0
- data/lib/karafka/setup/config.rb +48 -19
- data/lib/karafka/status.rb +35 -7
- data/lib/karafka/templates/karafka.rb.erb +14 -6
- data/lib/karafka/time_trackers/base.rb +1 -6
- data/lib/karafka/time_trackers/pause.rb +49 -19
- data/lib/karafka/time_trackers/poll.rb +7 -3
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +56 -3
- data/renovate.json +6 -0
- data.tar.gz.sig +0 -0
- metadata +189 -61
- metadata.gz.sig +0 -0
- data/bin/wait_for_kafka +0 -20
- data/certs/mensfeld.pem +0 -25
- data/lib/karafka/active_job/routing/extensions.rb +0 -31
- data/lib/karafka/contracts/consumer_group_topic.rb +0 -42
- data/lib/karafka/instrumentation.rb +0 -21
- data/lib/karafka/pro/base_consumer.rb +0 -82
- data/lib/karafka/pro/contracts/consumer_group.rb +0 -34
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +0 -33
- data/lib/karafka/pro/routing/topic_extensions.rb +0 -38
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# This Karafka component is a Pro component.
|
|
4
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
|
5
|
-
# repository and their usage requires commercial license agreement.
|
|
6
|
-
#
|
|
7
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
|
8
|
-
#
|
|
9
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
|
10
|
-
# your code to Maciej Mensfeld.
|
|
11
|
-
|
|
12
|
-
module Karafka
|
|
13
|
-
module Pro
|
|
14
|
-
module Contracts
|
|
15
|
-
# Contract for validating correct Pro components setup on a consumer group and topic levels
|
|
16
|
-
class ConsumerGroup < Base
|
|
17
|
-
virtual do |data, errors|
|
|
18
|
-
next unless errors.empty?
|
|
19
|
-
next unless data.key?(:topics)
|
|
20
|
-
|
|
21
|
-
fetched_errors = []
|
|
22
|
-
|
|
23
|
-
data.fetch(:topics).each do |topic|
|
|
24
|
-
ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
|
|
25
|
-
fetched_errors << [[topic, key].flatten, value]
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
fetched_errors
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
end
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# This Karafka component is a Pro component.
|
|
4
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
|
5
|
-
# repository and their usage requires commercial license agreement.
|
|
6
|
-
#
|
|
7
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
|
8
|
-
#
|
|
9
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
|
10
|
-
# your code to Maciej Mensfeld.
|
|
11
|
-
|
|
12
|
-
module Karafka
|
|
13
|
-
module Pro
|
|
14
|
-
module Contracts
|
|
15
|
-
# Contract for validating correct Pro components setup on a topic levels
|
|
16
|
-
class ConsumerGroupTopic < Base
|
|
17
|
-
configure do |config|
|
|
18
|
-
config.error_messages = YAML.safe_load(
|
|
19
|
-
File.read(
|
|
20
|
-
File.join(Karafka.gem_root, 'config', 'errors.yml')
|
|
21
|
-
)
|
|
22
|
-
).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
virtual do |data|
|
|
26
|
-
next if data[:consumer] < Karafka::Pro::BaseConsumer
|
|
27
|
-
|
|
28
|
-
[[%i[consumer], :consumer_format]]
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
end
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# This Karafka component is a Pro component.
|
|
4
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
|
5
|
-
# repository and their usage requires commercial license agreement.
|
|
6
|
-
#
|
|
7
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
|
8
|
-
#
|
|
9
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
|
10
|
-
# your code to Maciej Mensfeld.
|
|
11
|
-
|
|
12
|
-
module Karafka
|
|
13
|
-
module Pro
|
|
14
|
-
# Pro routing components
|
|
15
|
-
module Routing
|
|
16
|
-
# Routing extensions that allow to configure some extra PRO routing options
|
|
17
|
-
module TopicExtensions
|
|
18
|
-
class << self
|
|
19
|
-
# @param base [Class] class we extend
|
|
20
|
-
def included(base)
|
|
21
|
-
base.attr_accessor :long_running_job
|
|
22
|
-
base.attr_accessor :virtual_partitioner
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
# @return [Boolean] true if virtual partitioner is defined, false otherwise
|
|
27
|
-
def virtual_partitioner?
|
|
28
|
-
virtual_partitioner != nil
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# @return [Boolean] is a given job on a topic a long-running one
|
|
32
|
-
def long_running_job?
|
|
33
|
-
@long_running_job || false
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
end
|