karafka 2.1.4 → 2.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +28 -0
- data/Gemfile.lock +20 -20
- data/karafka.gemspec +2 -2
- data/lib/karafka/admin.rb +37 -4
- data/lib/karafka/base_consumer.rb +21 -5
- data/lib/karafka/connection/client.rb +118 -95
- data/lib/karafka/connection/rebalance_manager.rb +2 -4
- data/lib/karafka/errors.rb +4 -1
- data/lib/karafka/messages/builders/message.rb +0 -3
- data/lib/karafka/messages/seek.rb +3 -0
- data/lib/karafka/patches/rdkafka/bindings.rb +4 -6
- data/lib/karafka/pro/iterator/expander.rb +95 -0
- data/lib/karafka/pro/iterator/tpl_builder.rb +145 -0
- data/lib/karafka/pro/iterator.rb +2 -87
- data/lib/karafka/pro/processing/filters_applier.rb +1 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +4 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +2 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +2 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +2 -1
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +3 -1
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +8 -1
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +3 -1
- data/lib/karafka/pro/processing/virtual_offset_manager.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +14 -0
- data/lib/karafka/processing/strategies/default.rb +27 -11
- data/lib/karafka/railtie.rb +2 -2
- data/lib/karafka/setup/attributes_map.rb +1 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +10 -9
- metadata.gz.sig +0 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +0 -22
@@ -27,12 +27,7 @@ module Karafka
|
|
27
27
|
# Ignore earlier offsets than the one we already committed
|
28
28
|
return true if coordinator.seek_offset > message.offset
|
29
29
|
return false if revoked?
|
30
|
-
|
31
|
-
unless client.mark_as_consumed(message)
|
32
|
-
coordinator.revoke
|
33
|
-
|
34
|
-
return false
|
35
|
-
end
|
30
|
+
return revoked? unless client.mark_as_consumed(message)
|
36
31
|
|
37
32
|
coordinator.seek_offset = message.offset + 1
|
38
33
|
|
@@ -49,17 +44,38 @@ module Karafka
|
|
49
44
|
return true if coordinator.seek_offset > message.offset
|
50
45
|
return false if revoked?
|
51
46
|
|
52
|
-
unless client.mark_as_consumed!(message)
|
53
|
-
coordinator.revoke
|
54
|
-
|
55
|
-
return false
|
56
|
-
end
|
47
|
+
return revoked? unless client.mark_as_consumed!(message)
|
57
48
|
|
58
49
|
coordinator.seek_offset = message.offset + 1
|
59
50
|
|
60
51
|
true
|
61
52
|
end
|
62
53
|
|
54
|
+
# Triggers an async offset commit
|
55
|
+
#
|
56
|
+
# @param async [Boolean] should we use async (default) or sync commit
|
57
|
+
# @return [Boolean] true if we still own the partition.
|
58
|
+
# @note Due to its async nature, this may not fully represent the offset state in some
|
59
|
+
# edge cases (like for example going beyond max.poll.interval)
|
60
|
+
def commit_offsets(async: true)
|
61
|
+
# Do not commit if we already lost the assignment
|
62
|
+
return false if revoked?
|
63
|
+
return true if client.commit_offsets(async: async)
|
64
|
+
|
65
|
+
# This will once more check the librdkafka revocation status and will revoke the
|
66
|
+
# coordinator in case it was not revoked
|
67
|
+
revoked?
|
68
|
+
end
|
69
|
+
|
70
|
+
# Triggers a synchronous offsets commit to Kafka
|
71
|
+
#
|
72
|
+
# @return [Boolean] true if we still own the partition, false otherwise.
|
73
|
+
# @note This is fully synchronous, hence the result of this can be used in DB transactions
|
74
|
+
# etc as a way of making sure, that we still own the partition.
|
75
|
+
def commit_offsets!
|
76
|
+
commit_offsets(async: false)
|
77
|
+
end
|
78
|
+
|
63
79
|
# No actions needed for the standard flow here
|
64
80
|
def handle_before_enqueue
|
65
81
|
nil
|
data/lib/karafka/railtie.rb
CHANGED
@@ -70,11 +70,11 @@ if Karafka.rails?
|
|
70
70
|
# We can have many listeners, but it does not matter in which we will reload the code
|
71
71
|
# as long as all the consumers will be re-created as Rails reload is thread-safe
|
72
72
|
::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
|
73
|
-
# Reload code each time there is a change in the code
|
74
|
-
next unless Rails.application.reloaders.any?(&:updated?)
|
75
73
|
# If consumer persistence is enabled, no reason to reload because we will still keep
|
76
74
|
# old consumer instances in memory.
|
77
75
|
next if Karafka::App.config.consumer_persistence
|
76
|
+
# Reload code each time there is a change in the code
|
77
|
+
next unless Rails.application.reloaders.any?(&:updated?)
|
78
78
|
|
79
79
|
Rails.application.reloader.reload!
|
80
80
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-06-
|
38
|
+
date: 2023-06-29 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,20 +43,20 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.
|
46
|
+
version: 2.1.1
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
|
-
version:
|
49
|
+
version: 2.2.0
|
50
50
|
type: :runtime
|
51
51
|
prerelease: false
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.
|
56
|
+
version: 2.1.1
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
version:
|
59
|
+
version: 2.2.0
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: thor
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.
|
80
|
+
version: 2.6.2
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.
|
90
|
+
version: 2.6.2
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -214,7 +214,6 @@ files:
|
|
214
214
|
- lib/karafka/messages/parser.rb
|
215
215
|
- lib/karafka/messages/seek.rb
|
216
216
|
- lib/karafka/patches/rdkafka/bindings.rb
|
217
|
-
- lib/karafka/patches/rdkafka/consumer.rb
|
218
217
|
- lib/karafka/pro.rb
|
219
218
|
- lib/karafka/pro/active_job/consumer.rb
|
220
219
|
- lib/karafka/pro/active_job/dispatcher.rb
|
@@ -227,6 +226,8 @@ files:
|
|
227
226
|
- lib/karafka/pro/encryption/messages/parser.rb
|
228
227
|
- lib/karafka/pro/encryption/setup/config.rb
|
229
228
|
- lib/karafka/pro/iterator.rb
|
229
|
+
- lib/karafka/pro/iterator/expander.rb
|
230
|
+
- lib/karafka/pro/iterator/tpl_builder.rb
|
230
231
|
- lib/karafka/pro/loader.rb
|
231
232
|
- lib/karafka/pro/performance_tracker.rb
|
232
233
|
- lib/karafka/pro/processing/collapser.rb
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,22 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# Patches to external components
|
5
|
-
module Patches
|
6
|
-
# Rdkafka related patches
|
7
|
-
module Rdkafka
|
8
|
-
# Rdkafka::Consumer patches
|
9
|
-
module Consumer
|
10
|
-
# A method that allows us to get the native kafka producer name
|
11
|
-
# @return [String] producer instance name
|
12
|
-
# @note We need this to make sure that we allocate proper dispatched events only to
|
13
|
-
# callback listeners that should publish them
|
14
|
-
def name
|
15
|
-
@name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer
|