karafka 2.0.23 → 2.0.26

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +24 -3
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +52 -1
  6. data/Gemfile.lock +14 -12
  7. data/README.md +6 -4
  8. data/bin/integrations +8 -0
  9. data/bin/verify_license_integrity +35 -0
  10. data/config/{errors.yml → locales/errors.yml} +2 -1
  11. data/config/locales/pro_errors.yml +18 -0
  12. data/docker-compose.yml +3 -0
  13. data/karafka.gemspec +3 -3
  14. data/lib/karafka/active_job/job_options_contract.rb +1 -1
  15. data/lib/karafka/admin.rb +16 -14
  16. data/lib/karafka/app.rb +16 -4
  17. data/lib/karafka/base_consumer.rb +37 -7
  18. data/lib/karafka/connection/client.rb +21 -0
  19. data/lib/karafka/connection/consumer_group_coordinator.rb +7 -1
  20. data/lib/karafka/connection/listener.rb +5 -4
  21. data/lib/karafka/connection/listeners_batch.rb +6 -0
  22. data/lib/karafka/contracts/config.rb +1 -1
  23. data/lib/karafka/contracts/consumer_group.rb +1 -1
  24. data/lib/karafka/contracts/server_cli_options.rb +2 -1
  25. data/lib/karafka/contracts/topic.rb +13 -2
  26. data/lib/karafka/instrumentation/logger_listener.rb +50 -2
  27. data/lib/karafka/instrumentation/notifications.rb +17 -7
  28. data/lib/karafka/instrumentation/proctitle_listener.rb +7 -16
  29. data/lib/karafka/instrumentation/vendors/datadog/listener.rb +2 -2
  30. data/lib/karafka/messages/message.rb +14 -2
  31. data/lib/karafka/messages/parser.rb +14 -0
  32. data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
  33. data/lib/karafka/pro/encryption/cipher.rb +58 -0
  34. data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
  35. data/lib/karafka/pro/encryption/errors.rb +24 -0
  36. data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
  37. data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
  38. data/lib/karafka/pro/encryption/setup/config.rb +48 -0
  39. data/lib/karafka/pro/encryption.rb +47 -0
  40. data/lib/karafka/pro/loader.rb +22 -1
  41. data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +1 -1
  42. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +6 -1
  43. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +1 -1
  44. data/lib/karafka/pro/processing/strategies/default.rb +7 -1
  45. data/lib/karafka/pro/processing/strategies/dlq.rb +1 -1
  46. data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +1 -1
  47. data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +1 -1
  48. data/lib/karafka/pro/processing/strategies/dlq_mom.rb +1 -1
  49. data/lib/karafka/pro/processing/strategies/lrj.rb +6 -1
  50. data/lib/karafka/pro/processing/strategies/lrj_mom.rb +6 -1
  51. data/lib/karafka/pro/processing/strategies/mom.rb +1 -1
  52. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +2 -2
  53. data/lib/karafka/pro/routing/features/long_running_job/contract.rb +2 -2
  54. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +2 -2
  55. data/lib/karafka/process.rb +3 -1
  56. data/lib/karafka/processing/executor.rb +1 -1
  57. data/lib/karafka/processing/jobs_queue.rb +2 -2
  58. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  59. data/lib/karafka/processing/strategies/base.rb +5 -0
  60. data/lib/karafka/processing/strategies/default.rb +15 -1
  61. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  62. data/lib/karafka/processing/strategies/dlq_mom.rb +1 -1
  63. data/lib/karafka/processing/strategies/mom.rb +1 -1
  64. data/lib/karafka/processing/worker.rb +3 -1
  65. data/lib/karafka/railtie.rb +3 -0
  66. data/lib/karafka/routing/builder.rb +1 -1
  67. data/lib/karafka/routing/consumer_group.rb +3 -3
  68. data/lib/karafka/routing/consumer_mapper.rb +0 -10
  69. data/lib/karafka/routing/features/active_job/contract.rb +1 -1
  70. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +1 -1
  71. data/lib/karafka/routing/features/manual_offset_management/contract.rb +1 -1
  72. data/lib/karafka/routing/router.rb +12 -2
  73. data/lib/karafka/routing/subscription_group.rb +18 -1
  74. data/lib/karafka/routing/topic.rb +11 -0
  75. data/lib/karafka/runner.rb +1 -0
  76. data/lib/karafka/server.rb +27 -18
  77. data/lib/karafka/setup/config.rb +15 -2
  78. data/lib/karafka/status.rb +33 -9
  79. data/lib/karafka/templates/karafka.rb.erb +1 -2
  80. data/lib/karafka/time_trackers/base.rb +1 -6
  81. data/lib/karafka/time_trackers/pause.rb +5 -3
  82. data/lib/karafka/time_trackers/poll.rb +2 -2
  83. data/lib/karafka/version.rb +1 -1
  84. data/lib/karafka.rb +2 -0
  85. data.tar.gz.sig +0 -0
  86. metadata +18 -8
  87. metadata.gz.sig +0 -0
@@ -21,7 +21,7 @@ module Karafka
21
21
  def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
22
22
  proc_config = ::Karafka::App.config.internal.processing
23
23
 
24
- @id = SecureRandom.uuid
24
+ @id = SecureRandom.hex(6)
25
25
  @consumer_group_coordinator = consumer_group_coordinator
26
26
  @subscription_group = subscription_group
27
27
  @jobs_queue = jobs_queue
@@ -85,7 +85,7 @@ module Karafka
85
85
  # propagate this far.
86
86
  def fetch_loop
87
87
  # Run the main loop as long as we are not stopping or moving into quiet mode
88
- until Karafka::App.stopping? || Karafka::App.quieting?
88
+ until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
89
89
  Karafka.monitor.instrument(
90
90
  'connection.listener.fetch_loop',
91
91
  caller: self,
@@ -156,8 +156,9 @@ module Karafka
156
156
  # within this consumer group
157
157
  @consumer_group_coordinator.finish_work(id)
158
158
 
159
- # Wait if we're in the quiet mode
160
- wait_pinging(wait_until: -> { !Karafka::App.quieting? })
159
+ # Wait if we're in the process of finishing started work or finished all the work and
160
+ # just sitting and being quiet
161
+ wait_pinging(wait_until: -> { !(Karafka::App.quieting? || Karafka::App.quiet?) })
161
162
 
162
163
  # We need to wait until all the work in the whole consumer group (local to the process)
163
164
  # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
@@ -6,14 +6,20 @@ module Karafka
6
6
  class ListenersBatch
7
7
  include Enumerable
8
8
 
9
+ attr_reader :coordinators
10
+
9
11
  # @param jobs_queue [JobsQueue]
10
12
  # @return [ListenersBatch]
11
13
  def initialize(jobs_queue)
14
+ @coordinators = []
15
+
12
16
  @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
13
17
  consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
14
18
  subscription_groups.size
15
19
  )
16
20
 
21
+ @coordinators << consumer_group_coordinator
22
+
17
23
  subscription_groups.map do |subscription_group|
18
24
  Connection::Listener.new(
19
25
  consumer_group_coordinator,
@@ -12,7 +12,7 @@ module Karafka
12
12
  configure do |config|
13
13
  config.error_messages = YAML.safe_load(
14
14
  File.read(
15
- File.join(Karafka.gem_root, 'config', 'errors.yml')
15
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
16
16
  )
17
17
  ).fetch('en').fetch('validations').fetch('config')
18
18
  end
@@ -7,7 +7,7 @@ module Karafka
7
7
  configure do |config|
8
8
  config.error_messages = YAML.safe_load(
9
9
  File.read(
10
- File.join(Karafka.gem_root, 'config', 'errors.yml')
10
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
11
11
  )
12
12
  ).fetch('en').fetch('validations').fetch('consumer_group')
13
13
  end
@@ -7,7 +7,7 @@ module Karafka
7
7
  configure do |config|
8
8
  config.error_messages = YAML.safe_load(
9
9
  File.read(
10
- File.join(Karafka.gem_root, 'config', 'errors.yml')
10
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
11
11
  )
12
12
  ).fetch('en').fetch('validations').fetch('server_cli_options')
13
13
  end
@@ -80,6 +80,7 @@ module Karafka
80
80
  # Makes sure we have anything to subscribe to when we start the server
81
81
  virtual do |_, errors|
82
82
  next unless errors.empty?
83
+
83
84
  next unless Karafka::App.subscription_groups.empty?
84
85
 
85
86
  [[%i[topics], :topics_missing]]
@@ -7,12 +7,11 @@ module Karafka
7
7
  configure do |config|
8
8
  config.error_messages = YAML.safe_load(
9
9
  File.read(
10
- File.join(Karafka.gem_root, 'config', 'errors.yml')
10
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
11
11
  )
12
12
  ).fetch('en').fetch('validations').fetch('topic')
13
13
  end
14
14
 
15
- required(:consumer) { |val| !val.nil? }
16
15
  required(:deserializer) { |val| !val.nil? }
17
16
  required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
18
17
  required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
@@ -20,8 +19,20 @@ module Karafka
20
19
  required(:initial_offset) { |val| %w[earliest latest].include?(val) }
21
20
  required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
22
21
  required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
22
+ required(:active) { |val| [true, false].include?(val) }
23
23
  required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
24
24
 
25
+ # Consumer needs to be present only if topic is active
26
+ # We allow not to define consumer for non-active because they may be only used via admin
27
+ # api or other ways and not consumed with consumer
28
+ virtual do |data, errors|
29
+ next unless errors.empty?
30
+ next if data.fetch(:consumer)
31
+ next unless data.fetch(:active)
32
+
33
+ [[%w[consumer], :missing]]
34
+ end
35
+
25
36
  virtual do |data, errors|
26
37
  next unless errors.empty?
27
38
 
@@ -63,6 +63,49 @@ module Karafka
63
63
  info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
64
64
  end
65
65
 
66
+ # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
67
+ #
68
+ # @param event [Karafka::Core::Monitoring::Event] event details including payload
69
+ def on_client_pause(event)
70
+ topic = event[:topic]
71
+ partition = event[:partition]
72
+ offset = event[:offset]
73
+ client = event[:caller]
74
+
75
+ info <<~MSG.tr("\n", ' ').strip!
76
+ [#{client.id}] Pausing partition #{partition} of topic #{topic} on offset #{offset}
77
+ MSG
78
+ end
79
+
80
+ # Prints information about resuming of processing of a given topic partition
81
+ #
82
+ # @param event [Karafka::Core::Monitoring::Event] event details including payload
83
+ def on_client_resume(event)
84
+ topic = event[:topic]
85
+ partition = event[:partition]
86
+ client = event[:caller]
87
+
88
+ info <<~MSG.tr("\n", ' ').strip!
89
+ [#{client.id}] Resuming partition #{partition} of topic #{topic}
90
+ MSG
91
+ end
92
+
93
+ # Prints info about retry of processing after an error
94
+ #
95
+ # @param event [Karafka::Core::Monitoring::Event] event details including payload
96
+ def on_consumer_consuming_retry(event)
97
+ topic = event[:topic]
98
+ partition = event[:partition]
99
+ offset = event[:offset]
100
+ consumer = event[:caller]
101
+ timeout = event[:timeout]
102
+
103
+ info <<~MSG.tr("\n", ' ').strip!
104
+ [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
105
+ on partition #{partition} of topic #{topic} from offset #{offset}
106
+ MSG
107
+ end
108
+
66
109
  # Logs info about system signals that Karafka received and prints backtrace for threads in
67
110
  # case of ttin
68
111
  #
@@ -96,12 +139,17 @@ module Karafka
96
139
 
97
140
  return if Karafka.pro?
98
141
 
99
- info 'See LICENSE and the LGPL-3.0 for licensing details.'
142
+ info 'See LICENSE and the LGPL-3.0 for licensing details'
100
143
  end
101
144
 
102
145
  # @param _event [Karafka::Core::Monitoring::Event] event details including payload
103
146
  def on_app_quieting(_event)
104
- info 'Switching to quiet mode. New messages will not be processed.'
147
+ info 'Switching to quiet mode. New messages will not be processed'
148
+ end
149
+
150
+ # @param _event [Karafka::Core::Monitoring::Event] event details including payload
151
+ def on_app_quiet(_event)
152
+ info 'Reached quiet mode. No messages will be processed anymore'
105
153
  end
106
154
 
107
155
  # Logs info that we're going to stop the Karafka server.
@@ -20,27 +20,37 @@ module Karafka
20
20
  app.initialized
21
21
  app.running
22
22
  app.quieting
23
+ app.quiet
23
24
  app.stopping
24
25
  app.stopped
26
+ app.terminated
25
27
 
26
- consumer.consumed
27
- consumer.revoked
28
- consumer.shutdown
29
-
30
- process.notice_signal
28
+ client.pause
29
+ client.resume
31
30
 
32
31
  connection.listener.before_fetch_loop
33
32
  connection.listener.fetch_loop
34
33
  connection.listener.fetch_loop.received
35
34
 
35
+ consumer.consume
36
+ consumer.consumed
37
+ consumer.consuming.pause
38
+ consumer.consuming.retry
39
+ consumer.revoke
40
+ consumer.revoked
41
+ consumer.shutting_down
42
+ consumer.shutdown
43
+
36
44
  dead_letter_queue.dispatched
37
45
 
46
+ process.notice_signal
47
+
48
+ statistics.emitted
49
+
38
50
  worker.process
39
51
  worker.processed
40
52
  worker.completed
41
53
 
42
- statistics.emitted
43
-
44
54
  error.occurred
45
55
  ].freeze
46
56
 
@@ -4,22 +4,13 @@ module Karafka
4
4
  module Instrumentation
5
5
  # Listener that sets a proc title with a nice descriptive value
6
6
  class ProctitleListener
7
- # Updates proc title to an initializing one
8
- # @param _event [Karafka::Core::Monitoring::Event] event details including payload
9
- def on_app_initializing(_event)
10
- setproctitle('initializing')
11
- end
12
-
13
- # Updates proc title to a running one
14
- # @param _event [Karafka::Core::Monitoring::Event] event details including payload
15
- def on_app_running(_event)
16
- setproctitle('running')
17
- end
18
-
19
- # Updates proc title to a stopping one
20
- # @param _event [Karafka::Core::Monitoring::Event] event details including payload
21
- def on_app_stopping(_event)
22
- setproctitle('stopping')
7
+ Status::STATES.each_key do |state|
8
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
9
+ # Updates proc title to an appropriate state
10
+ def on_app_#{state}(_event)
11
+ setproctitle('#{state}')
12
+ end
13
+ RUBY
23
14
  end
24
15
 
25
16
  private
@@ -148,7 +148,7 @@ module Karafka
148
148
  jq_stats = event[:jobs_queue].statistics
149
149
 
150
150
  gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
151
- histogram('worker.processing', jq_stats[:processing], tags: default_tags)
151
+ histogram('worker.processing', jq_stats[:busy], tags: default_tags)
152
152
  histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
153
153
  end
154
154
 
@@ -158,7 +158,7 @@ module Karafka
158
158
  def on_worker_processed(event)
159
159
  jq_stats = event[:jobs_queue].statistics
160
160
 
161
- histogram('worker.processing', jq_stats[:processing], tags: default_tags)
161
+ histogram('worker.processing', jq_stats[:busy], tags: default_tags)
162
162
  end
163
163
 
164
164
  private
@@ -9,7 +9,19 @@ module Karafka
9
9
  class Message
10
10
  extend Forwardable
11
11
 
12
- attr_reader :raw_payload, :metadata
12
+ class << self
13
+ # @return [Object] general parser
14
+ # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
15
+ # via the config chain.
16
+ def parser
17
+ @parser ||= App.config.internal.messages.parser
18
+ end
19
+ end
20
+
21
+ attr_reader :metadata
22
+ # raw payload needs to be mutable as we want to have option to change it in the parser
23
+ # prior to the final deserialization
24
+ attr_accessor :raw_payload
13
25
 
14
26
  def_delegators :metadata, *Metadata.members
15
27
 
@@ -42,7 +54,7 @@ module Karafka
42
54
 
43
55
  # @return [Object] deserialized data
44
56
  def deserialize
45
- metadata.deserializer.call(self)
57
+ self.class.parser.call(self)
46
58
  end
47
59
  end
48
60
  end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Messages
5
+ # Default message parser. The only thing it does, is calling the deserializer
6
+ class Parser
7
+ # @param message [::Karafka::Messages::Message]
8
+ # @return [Object] deserialized payload
9
+ def call(message)
10
+ message.metadata.deserializer.call(message)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -20,7 +20,7 @@ module Karafka
20
20
  configure do |config|
21
21
  config.error_messages = YAML.safe_load(
22
22
  File.read(
23
- File.join(Karafka.gem_root, 'config', 'errors.yml')
23
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
24
24
  )
25
25
  ).fetch('en').fetch('validations').fetch('job_options')
26
26
  end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ # Cipher for encrypting and decrypting data
18
+ class Cipher
19
+ def initialize
20
+ @private_pems = {}
21
+ end
22
+
23
+ # Encrypts given string content with the public key
24
+ # @param content [String]
25
+ # @return [String]
26
+ def encrypt(content)
27
+ public_pem.public_encrypt(content)
28
+ end
29
+
30
+ # Decrypts provided content using `version` key
31
+ # @param version [String] encryption version
32
+ # @param content [String] encrypted content
33
+ # @return [String] decrypted content
34
+ def decrypt(version, content)
35
+ private_pem(version).private_decrypt(content)
36
+ end
37
+
38
+ private
39
+
40
+ # @return [::OpenSSL::PKey::RSA] rsa public key
41
+ def public_pem
42
+ @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
43
+ end
44
+
45
+ # @param version [String] version for which we want to get the rsa key
46
+ # @return [::OpenSSL::PKey::RSA] rsa private key
47
+ def private_pem(version)
48
+ return @private_pems[version] if @private_pems.key?(version)
49
+
50
+ key_string = ::Karafka::App.config.encryption.private_keys[version]
51
+ key_string || raise(Errors::PrivateKeyNotFound, version)
52
+
53
+ @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ # Encryption related contracts
18
+ module Contracts
19
+ # Makes sure, all the expected config is defined as it should be
20
+ class Config < ::Karafka::Contracts::Base
21
+ configure do |config|
22
+ config.error_messages = YAML.safe_load(
23
+ File.read(
24
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
+ )
26
+ ).fetch('en').fetch('validations').fetch('config')
27
+ end
28
+
29
+ nested(:encryption) do
30
+ required(:active) { |val| [true, false].include?(val) }
31
+ required(:version) { |val| val.is_a?(String) && !val.empty? }
32
+ required(:public_key) { |val| val.is_a?(String) }
33
+
34
+ required(:private_keys) do |val|
35
+ val.is_a?(Hash) &&
36
+ val.keys.all? { |key| key.is_a?(String) } &&
37
+ val.values.all? { |key| key.is_a?(String) }
38
+ end
39
+ end
40
+
41
+ # Public key validation
42
+ virtual do |data, errors|
43
+ next unless errors.empty?
44
+ next unless data.fetch(:encryption).fetch(:active)
45
+
46
+ key = OpenSSL::PKey::RSA.new(data.fetch(:encryption).fetch(:public_key))
47
+
48
+ next unless key.private?
49
+
50
+ [[%i[encryption public_key], :needs_to_be_public]]
51
+ rescue OpenSSL::PKey::RSAError
52
+ [[%i[encryption public_key], :invalid]]
53
+ end
54
+
55
+ # Private keys validation
56
+ virtual do |data, errors|
57
+ next unless errors.empty?
58
+ next unless data.fetch(:encryption).fetch(:active)
59
+
60
+ private_keys = data.fetch(:encryption).fetch(:private_keys)
61
+
62
+ # Keys may be empty for production only envs
63
+ next if private_keys.empty?
64
+
65
+ keys = private_keys.each_value.map do |key|
66
+ OpenSSL::PKey::RSA.new(key)
67
+ end
68
+
69
+ next if keys.all?(&:private?)
70
+
71
+ [[%i[encryption private_keys], :need_to_be_private]]
72
+ rescue OpenSSL::PKey::RSAError
73
+ [[%i[encryption private_keys], :invalid]]
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ # Encryption related errors
18
+ module Errors
19
+ # Raised when we have encountered encryption key with version we do not have
20
+ PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ # Encryption related messages components
18
+ module Messages
19
+ # Middleware for WaterDrop. It automatically encrypts messages payload.
20
+ # It is injected only if encryption is enabled.
21
+ class Middleware
22
+ # @param message [Hash] WaterDrop message hash
23
+ # @return [Hash] hash with encrypted payload and encryption version indicator
24
+ def call(message)
25
+ message[:headers] ||= {}
26
+ message[:headers]['encryption'] = version
27
+ message[:payload] = cipher.encrypt(message[:payload])
28
+ message
29
+ end
30
+
31
+ private
32
+
33
+ # @return [::Karafka::Pro::Encryption::Cipher]
34
+ def cipher
35
+ @cipher ||= ::Karafka::App.config.encryption.cipher
36
+ end
37
+
38
+ # @return [String] encryption version
39
+ def version
40
+ @version ||= ::Karafka::App.config.encryption.version
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ module Messages
18
+ # Pro parser that takes into consideration encryption usage
19
+ # @note There may be a case where someone decides not to encrypt data and we start getting
20
+ # unencrypted payloads. That is why we always rely on message headers for encryption
21
+ # indication.
22
+ class Parser < ::Karafka::Messages::Parser
23
+ # @param message [::Karafka::Messages::Message]
24
+ # @return [Object] deserialized payload
25
+ def call(message)
26
+ if active? && message.headers.key?('encryption')
27
+ # Decrypt raw payload so it can be handled by the default parser logic
28
+ message.raw_payload = cipher.decrypt(
29
+ message.headers['encryption'],
30
+ message.raw_payload
31
+ )
32
+ end
33
+
34
+ super(message)
35
+ end
36
+
37
+ private
38
+
39
+ # @return [::Karafka::Pro::Encryption::Cipher]
40
+ def cipher
41
+ @cipher ||= ::Karafka::App.config.encryption.cipher
42
+ end
43
+
44
+ # @return [Boolean] is encryption active
45
+ def active?
46
+ return @active unless @active.nil?
47
+
48
+ @active = ::Karafka::App.config.encryption.active
49
+
50
+ @active
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Encryption
17
+ # Setup and config related encryption components
18
+ module Setup
19
+ # Config for encryption
20
+ class Config
21
+ extend ::Karafka::Core::Configurable
22
+
23
+ # Should this feature be in use
24
+ setting(:active, default: false)
25
+
26
+ # Supporting versions allows us to be able to rotate private and public keys in case
27
+ # we would need this. We can increase the version, rotate and Karafka when decrypting
28
+ # will figure out proper private key based on the version
29
+ setting(:version, default: '1')
30
+
31
+ # We always support one public key for producing messages
32
+ # Public key needs to be always present even if we do not plan to produce messages from
33
+ # a Karafka process. This is because of the web-ui and potentially other cases like this
34
+ setting(:public_key, default: '')
35
+
36
+ # Private keys in pem format, where the key is the version and value is the key.
37
+ # This allows us to support key rotation
38
+ setting(:private_keys, default: {})
39
+
40
+ # Cipher used to encrypt and decrypt data
41
+ setting(:cipher, default: Encryption::Cipher.new)
42
+
43
+ configure
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end