karafka 2.1.4 → 2.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +28 -0
  4. data/Gemfile.lock +20 -20
  5. data/karafka.gemspec +2 -2
  6. data/lib/karafka/admin.rb +37 -4
  7. data/lib/karafka/base_consumer.rb +21 -5
  8. data/lib/karafka/connection/client.rb +118 -95
  9. data/lib/karafka/connection/rebalance_manager.rb +2 -4
  10. data/lib/karafka/errors.rb +4 -1
  11. data/lib/karafka/messages/builders/message.rb +0 -3
  12. data/lib/karafka/messages/seek.rb +3 -0
  13. data/lib/karafka/patches/rdkafka/bindings.rb +4 -6
  14. data/lib/karafka/pro/iterator/expander.rb +95 -0
  15. data/lib/karafka/pro/iterator/tpl_builder.rb +145 -0
  16. data/lib/karafka/pro/iterator.rb +2 -87
  17. data/lib/karafka/pro/processing/filters_applier.rb +1 -0
  18. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +3 -1
  19. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +3 -1
  20. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -1
  21. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +3 -1
  22. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +3 -1
  23. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +4 -1
  24. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +2 -2
  25. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +2 -2
  26. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +2 -1
  27. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +3 -1
  28. data/lib/karafka/pro/processing/strategies/ftr/default.rb +8 -1
  29. data/lib/karafka/pro/processing/strategies/lrj/default.rb +1 -1
  30. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +2 -2
  31. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +2 -2
  32. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +3 -1
  33. data/lib/karafka/pro/processing/virtual_offset_manager.rb +1 -1
  34. data/lib/karafka/processing/coordinator.rb +14 -0
  35. data/lib/karafka/processing/strategies/default.rb +27 -11
  36. data/lib/karafka/railtie.rb +2 -2
  37. data/lib/karafka/setup/attributes_map.rb +1 -0
  38. data/lib/karafka/version.rb +1 -1
  39. data.tar.gz.sig +0 -0
  40. metadata +10 -9
  41. metadata.gz.sig +0 -0
  42. data/lib/karafka/patches/rdkafka/consumer.rb +0 -22
@@ -27,12 +27,7 @@ module Karafka
27
27
  # Ignore earlier offsets than the one we already committed
28
28
  return true if coordinator.seek_offset > message.offset
29
29
  return false if revoked?
30
-
31
- unless client.mark_as_consumed(message)
32
- coordinator.revoke
33
-
34
- return false
35
- end
30
+ return revoked? unless client.mark_as_consumed(message)
36
31
 
37
32
  coordinator.seek_offset = message.offset + 1
38
33
 
@@ -49,17 +44,38 @@ module Karafka
49
44
  return true if coordinator.seek_offset > message.offset
50
45
  return false if revoked?
51
46
 
52
- unless client.mark_as_consumed!(message)
53
- coordinator.revoke
54
-
55
- return false
56
- end
47
+ return revoked? unless client.mark_as_consumed!(message)
57
48
 
58
49
  coordinator.seek_offset = message.offset + 1
59
50
 
60
51
  true
61
52
  end
62
53
 
54
+ # Triggers an async offset commit
55
+ #
56
+ # @param async [Boolean] should we use async (default) or sync commit
57
+ # @return [Boolean] true if we still own the partition.
58
+ # @note Due to its async nature, this may not fully represent the offset state in some
59
+ # edge cases (like for example going beyond max.poll.interval)
60
+ def commit_offsets(async: true)
61
+ # Do not commit if we already lost the assignment
62
+ return false if revoked?
63
+ return true if client.commit_offsets(async: async)
64
+
65
+ # This will once more check the librdkafka revocation status and will revoke the
66
+ # coordinator in case it was not revoked
67
+ revoked?
68
+ end
69
+
70
+ # Triggers a synchronous offsets commit to Kafka
71
+ #
72
+ # @return [Boolean] true if we still own the partition, false otherwise.
73
+ # @note This is fully synchronous, hence the result of this can be used in DB transactions
74
+ # etc as a way of making sure, that we still own the partition.
75
+ def commit_offsets!
76
+ commit_offsets(async: false)
77
+ end
78
+
63
79
  # No actions needed for the standard flow here
64
80
  def handle_before_enqueue
65
81
  nil
@@ -70,11 +70,11 @@ if Karafka.rails?
70
70
  # We can have many listeners, but it does not matter in which we will reload the code
71
71
  # as long as all the consumers will be re-created as Rails reload is thread-safe
72
72
  ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
73
- # Reload code each time there is a change in the code
74
- next unless Rails.application.reloaders.any?(&:updated?)
75
73
  # If consumer persistence is enabled, no reason to reload because we will still keep
76
74
  # old consumer instances in memory.
77
75
  next if Karafka::App.config.consumer_persistence
76
+ # Reload code each time there is a change in the code
77
+ next unless Rails.application.reloaders.any?(&:updated?)
78
78
 
79
79
  Rails.application.reloader.reload!
80
80
  end
@@ -47,6 +47,7 @@ module Karafka
47
47
  fetch.max.bytes
48
48
  fetch.message.max.bytes
49
49
  fetch.min.bytes
50
+ fetch.queue.backoff.ms
50
51
  fetch.wait.max.ms
51
52
  group.id
52
53
  group.instance.id
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.1.4'
6
+ VERSION = '2.1.6'
7
7
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.4
4
+ version: 2.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
36
36
  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
37
37
  -----END CERTIFICATE-----
38
- date: 2023-06-06 00:00:00.000000000 Z
38
+ date: 2023-06-29 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -43,20 +43,20 @@ dependencies:
43
43
  requirements:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: 2.0.13
46
+ version: 2.1.1
47
47
  - - "<"
48
48
  - !ruby/object:Gem::Version
49
- version: 3.0.0
49
+ version: 2.2.0
50
50
  type: :runtime
51
51
  prerelease: false
52
52
  version_requirements: !ruby/object:Gem::Requirement
53
53
  requirements:
54
54
  - - ">="
55
55
  - !ruby/object:Gem::Version
56
- version: 2.0.13
56
+ version: 2.1.1
57
57
  - - "<"
58
58
  - !ruby/object:Gem::Version
59
- version: 3.0.0
59
+ version: 2.2.0
60
60
  - !ruby/object:Gem::Dependency
61
61
  name: thor
62
62
  requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
77
77
  requirements:
78
78
  - - ">="
79
79
  - !ruby/object:Gem::Version
80
- version: 2.5.3
80
+ version: 2.6.2
81
81
  - - "<"
82
82
  - !ruby/object:Gem::Version
83
83
  version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
87
87
  requirements:
88
88
  - - ">="
89
89
  - !ruby/object:Gem::Version
90
- version: 2.5.3
90
+ version: 2.6.2
91
91
  - - "<"
92
92
  - !ruby/object:Gem::Version
93
93
  version: 3.0.0
@@ -214,7 +214,6 @@ files:
214
214
  - lib/karafka/messages/parser.rb
215
215
  - lib/karafka/messages/seek.rb
216
216
  - lib/karafka/patches/rdkafka/bindings.rb
217
- - lib/karafka/patches/rdkafka/consumer.rb
218
217
  - lib/karafka/pro.rb
219
218
  - lib/karafka/pro/active_job/consumer.rb
220
219
  - lib/karafka/pro/active_job/dispatcher.rb
@@ -227,6 +226,8 @@ files:
227
226
  - lib/karafka/pro/encryption/messages/parser.rb
228
227
  - lib/karafka/pro/encryption/setup/config.rb
229
228
  - lib/karafka/pro/iterator.rb
229
+ - lib/karafka/pro/iterator/expander.rb
230
+ - lib/karafka/pro/iterator/tpl_builder.rb
230
231
  - lib/karafka/pro/loader.rb
231
232
  - lib/karafka/pro/performance_tracker.rb
232
233
  - lib/karafka/pro/processing/collapser.rb
metadata.gz.sig CHANGED
Binary file
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Patches to external components
5
- module Patches
6
- # Rdkafka related patches
7
- module Rdkafka
8
- # Rdkafka::Consumer patches
9
- module Consumer
10
- # A method that allows us to get the native kafka producer name
11
- # @return [String] producer instance name
12
- # @note We need this to make sure that we allocate proper dispatched events only to
13
- # callback listeners that should publish them
14
- def name
15
- @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
16
- end
17
- end
18
- end
19
- end
20
- end
21
-
22
- ::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer