karafka 2.2.14 → 2.3.0.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +38 -12
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +23 -0
  6. data/Gemfile.lock +12 -12
  7. data/README.md +0 -2
  8. data/SECURITY.md +23 -0
  9. data/config/locales/errors.yml +7 -1
  10. data/config/locales/pro_errors.yml +22 -0
  11. data/docker-compose.yml +1 -1
  12. data/karafka.gemspec +2 -2
  13. data/lib/karafka/admin/acl.rb +287 -0
  14. data/lib/karafka/admin.rb +9 -13
  15. data/lib/karafka/app.rb +5 -3
  16. data/lib/karafka/base_consumer.rb +9 -1
  17. data/lib/karafka/cli/base.rb +1 -1
  18. data/lib/karafka/connection/client.rb +83 -76
  19. data/lib/karafka/connection/conductor.rb +28 -0
  20. data/lib/karafka/connection/listener.rb +159 -42
  21. data/lib/karafka/connection/listeners_batch.rb +5 -11
  22. data/lib/karafka/connection/manager.rb +72 -0
  23. data/lib/karafka/connection/messages_buffer.rb +12 -0
  24. data/lib/karafka/connection/proxy.rb +17 -0
  25. data/lib/karafka/connection/status.rb +75 -0
  26. data/lib/karafka/contracts/config.rb +14 -10
  27. data/lib/karafka/contracts/consumer_group.rb +9 -1
  28. data/lib/karafka/contracts/topic.rb +3 -1
  29. data/lib/karafka/errors.rb +13 -0
  30. data/lib/karafka/instrumentation/logger_listener.rb +3 -0
  31. data/lib/karafka/instrumentation/notifications.rb +13 -5
  32. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +31 -28
  33. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +20 -1
  34. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +15 -12
  35. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +39 -36
  36. data/lib/karafka/pro/base_consumer.rb +47 -0
  37. data/lib/karafka/pro/connection/manager.rb +300 -0
  38. data/lib/karafka/pro/connection/multiplexing/listener.rb +40 -0
  39. data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
  40. data/lib/karafka/pro/iterator.rb +1 -6
  41. data/lib/karafka/pro/loader.rb +14 -0
  42. data/lib/karafka/pro/processing/coordinator.rb +2 -1
  43. data/lib/karafka/pro/processing/executor.rb +37 -0
  44. data/lib/karafka/pro/processing/expansions_selector.rb +32 -0
  45. data/lib/karafka/pro/processing/jobs/periodic.rb +41 -0
  46. data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +32 -0
  47. data/lib/karafka/pro/processing/jobs_builder.rb +14 -3
  48. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +44 -0
  49. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +131 -0
  50. data/lib/karafka/pro/processing/offset_metadata/listener.rb +46 -0
  51. data/lib/karafka/pro/processing/schedulers/base.rb +39 -23
  52. data/lib/karafka/pro/processing/schedulers/default.rb +12 -14
  53. data/lib/karafka/pro/processing/strategies/default.rb +134 -1
  54. data/lib/karafka/pro/processing/strategies/dlq/default.rb +35 -0
  55. data/lib/karafka/pro/processing/strategies/vp/default.rb +59 -25
  56. data/lib/karafka/pro/processing/virtual_offset_manager.rb +41 -11
  57. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +2 -0
  58. data/lib/karafka/pro/routing/features/multiplexing/config.rb +38 -0
  59. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +114 -0
  60. data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +42 -0
  61. data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +38 -0
  62. data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +42 -0
  63. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +40 -0
  64. data/lib/karafka/pro/routing/features/multiplexing.rb +59 -0
  65. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +32 -0
  66. data/lib/karafka/pro/routing/features/non_blocking_job.rb +37 -0
  67. data/lib/karafka/pro/routing/features/offset_metadata/config.rb +33 -0
  68. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +42 -0
  69. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +65 -0
  70. data/lib/karafka/pro/routing/features/offset_metadata.rb +40 -0
  71. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +4 -0
  72. data/lib/karafka/pro/routing/features/patterns/detector.rb +18 -10
  73. data/lib/karafka/pro/routing/features/periodic_job/config.rb +37 -0
  74. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +44 -0
  75. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +94 -0
  76. data/lib/karafka/pro/routing/features/periodic_job.rb +27 -0
  77. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +1 -0
  78. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +1 -0
  79. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +7 -2
  80. data/lib/karafka/process.rb +5 -3
  81. data/lib/karafka/processing/coordinator.rb +5 -1
  82. data/lib/karafka/processing/executor.rb +16 -10
  83. data/lib/karafka/processing/executors_buffer.rb +19 -4
  84. data/lib/karafka/processing/schedulers/default.rb +3 -2
  85. data/lib/karafka/processing/strategies/default.rb +6 -0
  86. data/lib/karafka/processing/strategies/dlq.rb +36 -0
  87. data/lib/karafka/routing/builder.rb +12 -2
  88. data/lib/karafka/routing/consumer_group.rb +5 -5
  89. data/lib/karafka/routing/features/base.rb +44 -8
  90. data/lib/karafka/routing/features/dead_letter_queue/config.rb +6 -1
  91. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
  92. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +9 -2
  93. data/lib/karafka/routing/subscription_group.rb +2 -2
  94. data/lib/karafka/routing/subscription_groups_builder.rb +11 -2
  95. data/lib/karafka/routing/topic.rb +8 -10
  96. data/lib/karafka/runner.rb +13 -3
  97. data/lib/karafka/server.rb +5 -9
  98. data/lib/karafka/setup/config.rb +17 -0
  99. data/lib/karafka/status.rb +23 -14
  100. data/lib/karafka/templates/karafka.rb.erb +7 -0
  101. data/lib/karafka/time_trackers/partition_usage.rb +56 -0
  102. data/lib/karafka/version.rb +1 -1
  103. data.tar.gz.sig +0 -0
  104. metadata +42 -10
  105. metadata.gz.sig +0 -0
  106. data/lib/karafka/connection/consumer_group_coordinator.rb +0 -48
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class Multiplexing < Base
19
+ # Allows for multiplexing setup inside a consumer group definition
20
+ module Proxy
21
+ # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
22
+ # disabling dynamic multiplexing
23
+ # @param max [Integer] max multiplexing count
24
+ # @param boot [Integer] how many listeners should we start during boot by default
25
+ def multiplexing(min: nil, max: 1, boot: nil)
26
+ @target.current_subscription_group_details.merge!(
27
+ multiplexing_min: min || max,
28
+ multiplexing_max: max,
29
+ # Picks half of max by default as long as possible. Otherwise goes with min
30
+ multiplexing_boot: boot || [min || max, (max / 2)].max
31
+ )
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class Multiplexing < Base
19
+ # Adds methods needed for the multiplexing to work
20
+ module SubscriptionGroup
21
+ # @return [Config] multiplexing config
22
+ def multiplexing
23
+ @multiplexing ||= begin
24
+ max = @details.fetch(:multiplexing_max, 1)
25
+ min = @details.fetch(:multiplexing_min, max)
26
+ boot = @details.fetch(:multiplexing_boot, max / 2)
27
+ active = max > 1
28
+
29
+ Config.new(active: active, min: min, max: max, boot: boot)
30
+ end
31
+ end
32
+
33
+ # @return [Boolean] is multiplexing active
34
+ def multiplexing?
35
+ multiplexing.active?
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class Multiplexing < Base
19
+ # Expands the builder to multiply multiplexed groups
20
+ module SubscriptionGroupsBuilder
21
+ # Takes into consideration multiplexing and builds the more groups
22
+ #
23
+ # @param topics_array [Array<Routing::Topic>] group of topics that have the same
24
+ # settings and can use the same connection
25
+ # @return [Array<Array<Routing::Topics>>] expanded groups
26
+ def expand(topics_array)
27
+ factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
28
+
29
+ Array.new(factor) do |i|
30
+ ::Karafka::Routing::Topics.new(
31
+ i.zero? ? topics_array : topics_array.map(&:dup)
32
+ )
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ # Namespace for Pro routing enhancements
17
+ module Routing
18
+ # Namespace for additional Pro features
19
+ module Features
20
+ # Multiplexing allows for creating multiple subscription groups for the same topic inside
21
+ # of the same subscription group allowing for better parallelism with limited number
22
+ # of processes
23
+ class Multiplexing < Base
24
+ class << self
25
+ # @param _config [Karafka::Core::Configurable::Node] app config node
26
+ def pre_setup(_config)
27
+ # Make sure we use proper unique validator for topics definitions
28
+ ::Karafka::Contracts::ConsumerGroup.singleton_class.prepend(
29
+ Patches::Contracts::ConsumerGroup
30
+ )
31
+ end
32
+
33
+ # If needed installs the needed listener and initializes tracker
34
+ #
35
+ # @param _config [Karafka::Core::Configurable::Node] app config
36
+ def post_setup(_config)
37
+ ::Karafka::App.monitor.subscribe('app.running') do
38
+ # Do not install the manager and listener to control multiplexing unless there is
39
+ # multiplexing enabled and it is dynamic.
40
+ # We only need to control multiplexing when it is in a dynamic state
41
+ next unless ::Karafka::App
42
+ .subscription_groups
43
+ .values
44
+ .flat_map(&:itself)
45
+ .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
46
+
47
+ # Subscribe for events and possibility to manage via the Pro connection manager
48
+ # that supports multiplexing
49
+ ::Karafka.monitor.subscribe(
50
+ ::Karafka::Pro::Connection::Multiplexing::Listener.new
51
+ )
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class NonBlockingJob < Base
19
+ # Non-Blocking Jobs topic API extensions
20
+ module Topic
21
+ # @param args [Array] anything accepted by the `#long_running_job` API
22
+ def non_blocking_job(*args)
23
+ long_running_job(*args)
24
+ end
25
+
26
+ alias non_blocking non_blocking_job
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ # Namespace for Pro routing enhancements
17
+ module Routing
18
+ # Namespace for additional Pro features
19
+ module Features
20
+ # Non Blocking Job is just an alias for LRJ.
21
+ #
22
+ # We however have it as a separate feature because its use-case may vary from LRJ.
23
+ #
24
+ # While LRJ is used mainly for long-running jobs that would take more than max poll
25
+ # interval time, non-blocking can be applied to make sure that we do not wait with polling
26
+ # of different partitions and topics that are subscribed together.
27
+ #
28
+ # This effectively allows for better resources utilization
29
+ #
30
+ # All the underlying code is the same but use-case is different and this should be
31
+ # reflected in the routing, hence this "virtual" feature.
32
+ class NonBlockingJob < Base
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class OffsetMetadata < Base
19
+ # Config for commit metadata feature
20
+ Config = Struct.new(
21
+ :active,
22
+ :deserializer,
23
+ :cache,
24
+ keyword_init: true
25
+ ) do
26
+ alias_method :active?, :active
27
+ alias_method :cache?, :cache
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class OffsetMetadata < Base
19
+ # Namespace for offset metadata feature contracts
20
+ module Contracts
21
+ # Contract to validate configuration of the expiring feature
22
+ class Topic < Karafka::Contracts::Base
23
+ configure do |config|
24
+ config.error_messages = YAML.safe_load(
25
+ File.read(
26
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
27
+ )
28
+ ).fetch('en').fetch('validations').fetch('topic')
29
+ end
30
+
31
+ nested(:offset_metadata) do
32
+ required(:active) { |val| val == true }
33
+ required(:cache) { |val| [true, false].include?(val) }
34
+ required(:deserializer) { |val| val.respond_to?(:call) }
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ # This feature allows for saving and retrieving offset metadata with custom deserialization
19
+ # support. It allows for storing extra data during commits that can be then used to alter
20
+ # the processing flow after a rebalance.
21
+ #
22
+ # @note Because this feature has zero performance impact and makes no queries to Kafka
23
+ # unless requested, it is always enabled.
24
+ class OffsetMetadata < Base
25
+ # Empty string not to create it on each deserialization
26
+ EMPTY_STRING = ''
27
+
28
+ # Default deserializer just ensures we always get a string as without metadata by
29
+ # default it would be nil
30
+ STRING_DESERIALIZER = ->(raw_metadata) { raw_metadata || EMPTY_STRING }.freeze
31
+
32
+ private_constant :STRING_DESERIALIZER, :EMPTY_STRING
33
+
34
+ # Commit Metadata API extensions
35
+ module Topic
36
+ # @param cache [Boolean] should we cache the response until rebalance
37
+ # @param deserializer [#call] deserializer that will get raw data and should return
38
+ # deserialized metadata
39
+ # @return [Config] this feature config
40
+ def offset_metadata(cache: true, deserializer: STRING_DESERIALIZER)
41
+ @offset_metadata ||= Config.new(
42
+ active: true,
43
+ cache: cache,
44
+ deserializer: deserializer
45
+ )
46
+ end
47
+
48
+ # @return [true] is offset metadata active (it always is)
49
+ def offset_metadata?
50
+ offset_metadata.active?
51
+ end
52
+
53
+ # @return [Hash] topic with all its native configuration options plus offset metadata
54
+ # settings
55
+ def to_h
56
+ super.merge(
57
+ offset_metadata: offset_metadata.to_h
58
+ ).freeze
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ # Offset Metadata Support with a custom deserializer
19
+ class OffsetMetadata < Base
20
+ class << self
21
+ # If needed installs the needed listener and initializes tracker
22
+ #
23
+ # @param _config [Karafka::Core::Configurable::Node] app config
24
+ def post_setup(_config)
25
+ ::Karafka::App.monitor.subscribe('app.running') do
26
+ # Initialize the tracker prior to becoming multi-threaded
27
+ ::Karafka::Processing::InlineInsights::Tracker.instance
28
+
29
+ # Subscribe to the statistics reports and collect them
30
+ ::Karafka.monitor.subscribe(
31
+ ::Karafka::Pro::Processing::OffsetMetadata::Listener.new
32
+ )
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -46,7 +46,11 @@ module Karafka
46
46
  next unless errors.empty?
47
47
 
48
48
  regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
49
+ regexp_names = data[:patterns].map { |pattern| pattern.fetch(:name) }
49
50
 
51
+ # If all names are the same for the same regexp, it means its a multiplex and
52
+ # we can allow it
53
+ next if regexp_names.uniq.size == 1 && regexp_strings.uniq.size == 1
50
54
  next if regexp_strings.empty?
51
55
  next if regexp_strings.uniq.size == regexp_strings.size
52
56
 
@@ -22,6 +22,12 @@ module Karafka
22
22
  # @note This is NOT thread-safe and should run in a thread-safe context that warranties
23
23
  # that there won't be any race conditions
24
24
  class Detector
25
+ # Mutex for making sure that we do not modify same consumer group in runtime at the
26
+ # same time from multiple subscription groups if they operate in a multiplexed mode
27
+ MUTEX = Mutex.new
28
+
29
+ private_constant :MUTEX
30
+
25
31
  # Checks if the provided topic matches any of the patterns and when detected, expands
26
32
  # the routing with it.
27
33
  #
@@ -29,16 +35,18 @@ module Karafka
29
35
  # topics.
30
36
  # @param new_topic [String] new topic that we have detected
31
37
  def expand(sg_topics, new_topic)
32
- sg_topics
33
- .map(&:patterns)
34
- .select(&:active?)
35
- .select(&:matcher?)
36
- .map(&:pattern)
37
- .then { |pts| pts.empty? ? return : pts }
38
- .then { |pts| Patterns.new(pts) }
39
- .find(new_topic)
40
- .then { |pattern| pattern || return }
41
- .then { |pattern| install(pattern, new_topic, sg_topics) }
38
+ MUTEX.synchronize do
39
+ sg_topics
40
+ .map(&:patterns)
41
+ .select(&:active?)
42
+ .select(&:matcher?)
43
+ .map(&:pattern)
44
+ .then { |pts| pts.empty? ? return : pts }
45
+ .then { |pts| Patterns.new(pts) }
46
+ .find(new_topic)
47
+ .then { |pattern| pattern || return }
48
+ .then { |pattern| install(pattern, new_topic, sg_topics) }
49
+ end
42
50
  end
43
51
 
44
52
  private
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class PeriodicJob < Base
19
+ # Config for periodics topics feature
20
+ Config = Struct.new(
21
+ :active,
22
+ :during_pause,
23
+ :during_retry,
24
+ :interval,
25
+ :materialized,
26
+ keyword_init: true
27
+ ) do
28
+ alias_method :active?, :active
29
+ alias_method :during_pause?, :during_pause
30
+ alias_method :during_retry?, :during_retry
31
+ alias_method :materialized?, :materialized
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class PeriodicJob < Base
19
+ # Namespace for periodics messages contracts
20
+ module Contracts
21
+ # Contract to validate configuration of the periodics feature
22
+ class Topic < Karafka::Contracts::Base
23
+ configure do |config|
24
+ config.error_messages = YAML.safe_load(
25
+ File.read(
26
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
27
+ )
28
+ ).fetch('en').fetch('validations').fetch('topic')
29
+ end
30
+
31
+ nested(:periodic_job) do
32
+ required(:active) { |val| [true, false].include?(val) }
33
+ required(:interval) { |val| val.is_a?(Integer) && val >= 100 }
34
+ required(:during_pause) { |val| [true, false].include?(val) }
35
+ required(:during_retry) { |val| [true, false].include?(val) }
36
+ required(:materialized) { |val| [true, false].include?(val) }
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Routing
17
+ module Features
18
+ class PeriodicJob < Base
19
+ # Periodic topic action flows extensions
20
+ module Topic
21
+ # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
22
+ # poll where messages were not received.
23
+ # @param active [Boolean] should ticking happen for this topic assignments.
24
+ # @param interval [Integer] minimum interval to run periodic jobs on given topic.
25
+ # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
26
+ # It is set to `nil` by default allowing for detection when this value is not
27
+ # configured but should be built dynamically based on LRJ status.
28
+ # @param during_retry [Boolean, nil] Should we run when there was an error and we are
29
+ # in a retry flow. Please note that for this to work, `during_pause` also needs to be
30
+ # set to true as errors retry happens after pause.
31
+ def periodic_job(
32
+ active = false,
33
+ interval: nil,
34
+ during_pause: nil,
35
+ during_retry: nil
36
+ )
37
+ @periodic_job ||= begin
38
+ # Set to active if any of the values was configured
39
+ active = true unless interval.nil?
40
+ active = true unless during_pause.nil?
41
+ active = true unless during_retry.nil?
42
+ # Default is not to retry during retry flow
43
+ during_retry = false if during_retry.nil?
44
+
45
+ # If no interval, use default
46
+ interval ||= ::Karafka::App.config.internal.tick_interval
47
+
48
+ Config.new(
49
+ active: active,
50
+ interval: interval,
51
+ during_pause: during_pause,
52
+ during_retry: during_retry,
53
+ # This is internal setting for state management, not part of the configuration
54
+ # Do not overwrite.
55
+ # If `during_pause` is explicit, we do not select it based on LRJ setup and we
56
+ # consider if fully ready out of the box
57
+ materialized: !during_pause.nil?
58
+ )
59
+ end
60
+
61
+ return @periodic_job if @periodic_job.materialized?
62
+ return @periodic_job unless @long_running_job
63
+
64
+ # If not configured in any way, we want not to process during pause for LRJ.
65
+ # LRJ pauses by default when processing and during this time we do not want to
66
+ # tick at all. This prevents us from running periodic jobs while LRJ jobs are
67
+ # running. This of course has a side effect of not running when paused for any
68
+ # other reason but it is a compromise in the default settings
69
+ @periodic_job.during_pause = !long_running_job?
70
+ @periodic_job.materialized = true
71
+
72
+ @periodic_job
73
+ end
74
+
75
+ alias periodic periodic_job
76
+
77
+ # @return [Boolean] is periodics active
78
+ def periodic_job?
79
+ periodic_job.active?
80
+ end
81
+
82
+ # @return [Hash] topic with all its native configuration options plus periodics flows
83
+ # settings
84
+ def to_h
85
+ super.merge(
86
+ periodic_job: periodic_job.to_h
87
+ ).freeze
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
94
+ end