karafka 2.0.6 → 2.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +1 -1
- data/config/errors.yml +2 -0
- data/lib/karafka/pro/active_job/consumer.rb +1 -1
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +21 -1
- data/lib/karafka/pro/loader.rb +1 -1
- data/lib/karafka/pro/processing/partitioner.rb +3 -5
- data/lib/karafka/pro/routing/topic_extensions.rb +41 -5
- data/lib/karafka/routing/proxy.rb +9 -16
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +2 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0abed3f97a58be6b48f640468f7d7e6d48bc0960596b21d022b4616dd047be28
|
4
|
+
data.tar.gz: 48143253beee640e25e47a81474767c179e715e855d6173b59566483a57af5a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9c9f8c170ac82fc0f1eb6ea41698dcd82cc525006931a59443d004c94eb18b56ffcb67eb1eb45fcc1fd557fee22e6e63ceb7a8a001245469e3e574d87c88c8e8
|
7
|
+
data.tar.gz: 47bc7e7dfe5ca3d503a3cb18da4e4b95c076197dc26b5633195e169d3f4d94da4effaf27bd4360ddff1481031b1ee20f61e465e24f6984570f6067ca4fbd51ea
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 2.0.7 (Unreleased)
|
4
|
+
- [Breaking change] Redefine the Virtual Partitions routing DSL to accept concurrency
|
5
|
+
- Allow for `concurrency` setting in Virtual Partitions to extend or limit number of jobs per regular partition. This allows to make sure, we do not use all the threads on virtual partitions jobs
|
6
|
+
- Allow for creation of as many Virtual Partitions as needed, without taking global `concurrency` into consideration
|
7
|
+
|
3
8
|
## 2.0.6 (2022-09-02)
|
4
9
|
- Improve client closing.
|
5
10
|
- Fix for: Multiple LRJ topics fetched concurrently block ability for LRJ to kick in (#1002)
|
data/Gemfile.lock
CHANGED
data/config/errors.yml
CHANGED
@@ -55,3 +55,5 @@ en:
|
|
55
55
|
|
56
56
|
pro_consumer_group_topic:
|
57
57
|
consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
|
58
|
+
virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
|
59
|
+
virtual_partitions.concurrency_format: needs to be equl or more than 1
|
@@ -22,11 +22,31 @@ module Karafka
|
|
22
22
|
).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
|
23
23
|
end
|
24
24
|
|
25
|
-
|
25
|
+
nested(:virtual_partitions) do
|
26
|
+
required(:active) { |val| [true, false].include?(val) }
|
27
|
+
required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
|
28
|
+
required(:concurrency) { |val| val.is_a?(Integer) && val >= 1 }
|
29
|
+
end
|
30
|
+
|
31
|
+
virtual do |data, errors|
|
32
|
+
next unless errors.empty?
|
26
33
|
next if data[:consumer] < Karafka::Pro::BaseConsumer
|
27
34
|
|
28
35
|
[[%i[consumer], :consumer_format]]
|
29
36
|
end
|
37
|
+
|
38
|
+
# When virtual partitions are defined, partitioner needs to respond to `#call` and it
|
39
|
+
# cannot be nil
|
40
|
+
virtual do |data, errors|
|
41
|
+
next unless errors.empty?
|
42
|
+
|
43
|
+
virtual_partitions = data[:virtual_partitions]
|
44
|
+
|
45
|
+
next unless virtual_partitions[:active]
|
46
|
+
next if virtual_partitions[:partitioner].respond_to?(:call)
|
47
|
+
|
48
|
+
[[%i[virtual_partitions partitioner], :respond_to_call]]
|
49
|
+
end
|
30
50
|
end
|
31
51
|
end
|
32
52
|
end
|
data/lib/karafka/pro/loader.rb
CHANGED
@@ -67,7 +67,7 @@ module Karafka
|
|
67
67
|
|
68
68
|
# Loads routing extensions
|
69
69
|
def load_routing_extensions
|
70
|
-
::Karafka::Routing::Topic.
|
70
|
+
::Karafka::Routing::Topic.prepend(Routing::TopicExtensions)
|
71
71
|
::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
|
72
72
|
end
|
73
73
|
end
|
@@ -21,17 +21,15 @@ module Karafka
|
|
21
21
|
def call(topic, messages)
|
22
22
|
ktopic = @subscription_group.topics.find(topic)
|
23
23
|
|
24
|
-
@concurrency ||= ::Karafka::App.config.concurrency
|
25
|
-
|
26
24
|
# We only partition work if we have a virtual partitioner and more than one thread to
|
27
25
|
# process the data. With one thread it is not worth partitioning the work as the work
|
28
26
|
# itself will be assigned to one thread (pointless work)
|
29
|
-
if ktopic.
|
27
|
+
if ktopic.virtual_partitions? && ktopic.virtual_partitions.concurrency > 1
|
30
28
|
# We need to reduce it to number of threads, so the group_id is not a direct effect
|
31
29
|
# of the end user action. Otherwise the persistence layer for consumers would cache
|
32
30
|
# it forever and it would cause memory leaks
|
33
31
|
groupings = messages
|
34
|
-
.group_by { |msg| ktopic.
|
32
|
+
.group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
|
35
33
|
.values
|
36
34
|
|
37
35
|
# Reduce the max concurrency to a size that matches the concurrency
|
@@ -41,7 +39,7 @@ module Karafka
|
|
41
39
|
# The algorithm here is simple, we assume that the most costly in terms of processing,
|
42
40
|
# will be processing of the biggest group and we reduce the smallest once to have
|
43
41
|
# max of groups equal to concurrency
|
44
|
-
while groupings.size >
|
42
|
+
while groupings.size > ktopic.virtual_partitions.concurrency
|
45
43
|
groupings.sort_by! { |grouping| -grouping.size }
|
46
44
|
|
47
45
|
# Offset order needs to be maintained for virtual partitions
|
@@ -15,23 +15,59 @@ module Karafka
|
|
15
15
|
module Routing
|
16
16
|
# Routing extensions that allow to configure some extra PRO routing options
|
17
17
|
module TopicExtensions
|
18
|
+
# Internal representation of the virtual partitions settings and configuration
|
19
|
+
# This allows us to abstract away things in a nice manner
|
20
|
+
#
|
21
|
+
# For features with more options than just on/off we use this approach as it simplifies
|
22
|
+
# the code. We do not use it for all not to create unneeded complexity
|
23
|
+
VirtualPartitions = Struct.new(
|
24
|
+
:active,
|
25
|
+
:partitioner,
|
26
|
+
:concurrency,
|
27
|
+
keyword_init: true
|
28
|
+
) { alias_method :active?, :active }
|
29
|
+
|
18
30
|
class << self
|
19
31
|
# @param base [Class] class we extend
|
20
|
-
def
|
32
|
+
def prepended(base)
|
21
33
|
base.attr_accessor :long_running_job
|
22
|
-
base.attr_accessor :virtual_partitioner
|
23
34
|
end
|
24
35
|
end
|
25
36
|
|
26
|
-
# @
|
27
|
-
|
28
|
-
|
37
|
+
# @param concurrency [Integer] max number of virtual partitions that can come out of the
|
38
|
+
# single distribution flow. When set to more than the Karafka threading, will create
|
39
|
+
# more work than workers. When less, can ensure we have spare resources to process other
|
40
|
+
# things in parallel.
|
41
|
+
# @param partitioner [nil, #call] nil or callable partitioner
|
42
|
+
# @return [VirtualPartitions] method that allows to set the virtual partitions details
|
43
|
+
# during the routing configuration and then allows to retrieve it
|
44
|
+
def virtual_partitions(
|
45
|
+
concurrency: Karafka::App.config.concurrency,
|
46
|
+
partitioner: nil
|
47
|
+
)
|
48
|
+
@virtual_partitions ||= VirtualPartitions.new(
|
49
|
+
active: !partitioner.nil?,
|
50
|
+
concurrency: concurrency,
|
51
|
+
partitioner: partitioner
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
# @return [Boolean] are virtual partitions enabled for given topic
|
56
|
+
def virtual_partitions?
|
57
|
+
virtual_partitions.active?
|
29
58
|
end
|
30
59
|
|
31
60
|
# @return [Boolean] is a given job on a topic a long-running one
|
32
61
|
def long_running_job?
|
33
62
|
@long_running_job || false
|
34
63
|
end
|
64
|
+
|
65
|
+
# @return [Hash] hash with topic details and the extensions details
|
66
|
+
def to_h
|
67
|
+
super.merge(
|
68
|
+
virtual_partitions: virtual_partitions.to_h
|
69
|
+
)
|
70
|
+
end
|
35
71
|
end
|
36
72
|
end
|
37
73
|
end
|
@@ -7,15 +7,6 @@ module Karafka
|
|
7
7
|
class Proxy
|
8
8
|
attr_reader :target
|
9
9
|
|
10
|
-
# We should proxy only non ? and = methods as we want to have a regular dsl
|
11
|
-
IGNORED_POSTFIXES = %w[
|
12
|
-
?
|
13
|
-
=
|
14
|
-
!
|
15
|
-
].freeze
|
16
|
-
|
17
|
-
private_constant :IGNORED_POSTFIXES
|
18
|
-
|
19
10
|
# @param target [Object] target object to which we proxy any DSL call
|
20
11
|
# @param block [Proc] block that we want to evaluate in the proxy context
|
21
12
|
def initialize(target, &block)
|
@@ -25,21 +16,23 @@ module Karafka
|
|
25
16
|
|
26
17
|
# Translates the no "=" DSL of routing into elements assignments on target
|
27
18
|
# @param method_name [Symbol] name of the missing method
|
28
|
-
|
29
|
-
# @param block [Proc] block provided to the method
|
30
|
-
def method_missing(method_name, *arguments, &block)
|
19
|
+
def method_missing(method_name, ...)
|
31
20
|
return super unless respond_to_missing?(method_name)
|
32
21
|
|
33
|
-
@target.
|
22
|
+
if @target.respond_to?(:"#{method_name}=")
|
23
|
+
@target.public_send(:"#{method_name}=", ...)
|
24
|
+
else
|
25
|
+
@target.public_send(method_name, ...)
|
26
|
+
end
|
34
27
|
end
|
35
28
|
|
36
29
|
# Tells whether or not a given element exists on the target
|
37
30
|
# @param method_name [Symbol] name of the missing method
|
38
31
|
# @param include_private [Boolean] should we include private in the check as well
|
39
32
|
def respond_to_missing?(method_name, include_private = false)
|
40
|
-
|
41
|
-
|
42
|
-
|
33
|
+
@target.respond_to?(:"#{method_name}=", include_private) ||
|
34
|
+
@target.respond_to?(method_name, include_private) ||
|
35
|
+
super
|
43
36
|
end
|
44
37
|
end
|
45
38
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2022-09-
|
38
|
+
date: 2022-09-05 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
metadata.gz.sig
CHANGED
Binary file
|