karafka 1.4.13 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +3 -3
  3. data/.github/workflows/ci.yml +85 -30
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +268 -7
  6. data/CONTRIBUTING.md +10 -19
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +44 -87
  9. data/LICENSE +17 -0
  10. data/LICENSE-COMM +89 -0
  11. data/LICENSE-LGPL +165 -0
  12. data/README.md +44 -48
  13. data/bin/benchmarks +85 -0
  14. data/bin/create_token +22 -0
  15. data/bin/integrations +237 -0
  16. data/bin/karafka +4 -0
  17. data/bin/scenario +29 -0
  18. data/bin/stress_many +13 -0
  19. data/bin/stress_one +13 -0
  20. data/bin/wait_for_kafka +20 -0
  21. data/certs/karafka-pro.pem +11 -0
  22. data/config/errors.yml +55 -40
  23. data/docker-compose.yml +39 -3
  24. data/karafka.gemspec +11 -17
  25. data/lib/active_job/karafka.rb +21 -0
  26. data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
  27. data/lib/karafka/active_job/consumer.rb +26 -0
  28. data/lib/karafka/active_job/dispatcher.rb +38 -0
  29. data/lib/karafka/active_job/job_extensions.rb +34 -0
  30. data/lib/karafka/active_job/job_options_contract.rb +21 -0
  31. data/lib/karafka/active_job/routing/extensions.rb +31 -0
  32. data/lib/karafka/app.rb +15 -20
  33. data/lib/karafka/base_consumer.rb +181 -31
  34. data/lib/karafka/cli/base.rb +4 -4
  35. data/lib/karafka/cli/info.rb +43 -9
  36. data/lib/karafka/cli/install.rb +19 -10
  37. data/lib/karafka/cli/server.rb +17 -42
  38. data/lib/karafka/cli.rb +4 -11
  39. data/lib/karafka/connection/client.rb +385 -90
  40. data/lib/karafka/connection/listener.rb +246 -38
  41. data/lib/karafka/connection/listeners_batch.rb +24 -0
  42. data/lib/karafka/connection/messages_buffer.rb +84 -0
  43. data/lib/karafka/connection/pauses_manager.rb +46 -0
  44. data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
  45. data/lib/karafka/connection/rebalance_manager.rb +78 -0
  46. data/lib/karafka/contracts/base.rb +17 -0
  47. data/lib/karafka/contracts/config.rb +88 -11
  48. data/lib/karafka/contracts/consumer_group.rb +21 -189
  49. data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
  50. data/lib/karafka/contracts/server_cli_options.rb +19 -18
  51. data/lib/karafka/contracts.rb +1 -1
  52. data/lib/karafka/env.rb +46 -0
  53. data/lib/karafka/errors.rb +21 -21
  54. data/lib/karafka/helpers/async.rb +33 -0
  55. data/lib/karafka/helpers/colorize.rb +20 -0
  56. data/lib/karafka/helpers/multi_delegator.rb +2 -2
  57. data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
  58. data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
  59. data/lib/karafka/instrumentation/logger_listener.rb +164 -0
  60. data/lib/karafka/instrumentation/monitor.rb +13 -61
  61. data/lib/karafka/instrumentation/notifications.rb +52 -0
  62. data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
  63. data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
  64. data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
  65. data/lib/karafka/instrumentation.rb +21 -0
  66. data/lib/karafka/licenser.rb +75 -0
  67. data/lib/karafka/messages/batch_metadata.rb +45 -0
  68. data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
  69. data/lib/karafka/messages/builders/message.rb +39 -0
  70. data/lib/karafka/messages/builders/messages.rb +32 -0
  71. data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
  72. data/lib/karafka/messages/messages.rb +64 -0
  73. data/lib/karafka/{params → messages}/metadata.rb +4 -6
  74. data/lib/karafka/messages/seek.rb +9 -0
  75. data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
  76. data/lib/karafka/pro/active_job/consumer.rb +46 -0
  77. data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
  78. data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
  79. data/lib/karafka/pro/base_consumer.rb +82 -0
  80. data/lib/karafka/pro/contracts/base.rb +21 -0
  81. data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
  82. data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
  83. data/lib/karafka/pro/loader.rb +76 -0
  84. data/lib/karafka/pro/performance_tracker.rb +80 -0
  85. data/lib/karafka/pro/processing/coordinator.rb +72 -0
  86. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
  87. data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
  88. data/lib/karafka/pro/processing/partitioner.rb +60 -0
  89. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  90. data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
  91. data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
  92. data/lib/karafka/pro.rb +13 -0
  93. data/lib/karafka/process.rb +1 -0
  94. data/lib/karafka/processing/coordinator.rb +88 -0
  95. data/lib/karafka/processing/coordinators_buffer.rb +54 -0
  96. data/lib/karafka/processing/executor.rb +118 -0
  97. data/lib/karafka/processing/executors_buffer.rb +88 -0
  98. data/lib/karafka/processing/jobs/base.rb +51 -0
  99. data/lib/karafka/processing/jobs/consume.rb +42 -0
  100. data/lib/karafka/processing/jobs/revoked.rb +22 -0
  101. data/lib/karafka/processing/jobs/shutdown.rb +23 -0
  102. data/lib/karafka/processing/jobs_builder.rb +29 -0
  103. data/lib/karafka/processing/jobs_queue.rb +144 -0
  104. data/lib/karafka/processing/partitioner.rb +22 -0
  105. data/lib/karafka/processing/result.rb +29 -0
  106. data/lib/karafka/processing/scheduler.rb +22 -0
  107. data/lib/karafka/processing/worker.rb +88 -0
  108. data/lib/karafka/processing/workers_batch.rb +27 -0
  109. data/lib/karafka/railtie.rb +113 -0
  110. data/lib/karafka/routing/builder.rb +15 -24
  111. data/lib/karafka/routing/consumer_group.rb +11 -19
  112. data/lib/karafka/routing/consumer_mapper.rb +1 -2
  113. data/lib/karafka/routing/router.rb +1 -1
  114. data/lib/karafka/routing/subscription_group.rb +53 -0
  115. data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
  116. data/lib/karafka/routing/topic.rb +61 -24
  117. data/lib/karafka/routing/topics.rb +38 -0
  118. data/lib/karafka/runner.rb +51 -0
  119. data/lib/karafka/serialization/json/deserializer.rb +6 -15
  120. data/lib/karafka/server.rb +67 -26
  121. data/lib/karafka/setup/config.rb +147 -175
  122. data/lib/karafka/status.rb +14 -5
  123. data/lib/karafka/templates/example_consumer.rb.erb +16 -0
  124. data/lib/karafka/templates/karafka.rb.erb +15 -51
  125. data/lib/karafka/time_trackers/base.rb +19 -0
  126. data/lib/karafka/time_trackers/pause.rb +92 -0
  127. data/lib/karafka/time_trackers/poll.rb +65 -0
  128. data/lib/karafka/version.rb +1 -1
  129. data/lib/karafka.rb +38 -17
  130. data.tar.gz.sig +0 -0
  131. metadata +118 -120
  132. metadata.gz.sig +0 -0
  133. data/MIT-LICENCE +0 -18
  134. data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
  135. data/lib/karafka/attributes_map.rb +0 -63
  136. data/lib/karafka/backends/inline.rb +0 -16
  137. data/lib/karafka/base_responder.rb +0 -226
  138. data/lib/karafka/cli/flow.rb +0 -48
  139. data/lib/karafka/cli/missingno.rb +0 -19
  140. data/lib/karafka/code_reloader.rb +0 -67
  141. data/lib/karafka/connection/api_adapter.rb +0 -158
  142. data/lib/karafka/connection/batch_delegator.rb +0 -55
  143. data/lib/karafka/connection/builder.rb +0 -23
  144. data/lib/karafka/connection/message_delegator.rb +0 -36
  145. data/lib/karafka/consumers/batch_metadata.rb +0 -10
  146. data/lib/karafka/consumers/callbacks.rb +0 -71
  147. data/lib/karafka/consumers/includer.rb +0 -64
  148. data/lib/karafka/consumers/responders.rb +0 -24
  149. data/lib/karafka/consumers/single_params.rb +0 -15
  150. data/lib/karafka/contracts/responder_usage.rb +0 -54
  151. data/lib/karafka/fetcher.rb +0 -42
  152. data/lib/karafka/helpers/class_matcher.rb +0 -88
  153. data/lib/karafka/helpers/config_retriever.rb +0 -46
  154. data/lib/karafka/helpers/inflector.rb +0 -26
  155. data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
  156. data/lib/karafka/params/batch_metadata.rb +0 -26
  157. data/lib/karafka/params/builders/batch_metadata.rb +0 -30
  158. data/lib/karafka/params/builders/params.rb +0 -38
  159. data/lib/karafka/params/builders/params_batch.rb +0 -25
  160. data/lib/karafka/params/params_batch.rb +0 -60
  161. data/lib/karafka/patches/ruby_kafka.rb +0 -47
  162. data/lib/karafka/persistence/client.rb +0 -29
  163. data/lib/karafka/persistence/consumers.rb +0 -45
  164. data/lib/karafka/persistence/topics.rb +0 -48
  165. data/lib/karafka/responders/builder.rb +0 -36
  166. data/lib/karafka/responders/topic.rb +0 -55
  167. data/lib/karafka/routing/topic_mapper.rb +0 -53
  168. data/lib/karafka/serialization/json/serializer.rb +0 -31
  169. data/lib/karafka/setup/configurators/water_drop.rb +0 -36
  170. data/lib/karafka/templates/application_responder.rb.erb +0 -11
@@ -2,19 +2,96 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Contract with validation rules for Karafka configuration details
5
+ # Contract with validation rules for Karafka configuration details.
6
+ #
6
7
  # @note There are many more configuration options inside of the
7
- # Karafka::Setup::Config model, but we don't validate them here as they are
8
+ # `Karafka::Setup::Config` model, but we don't validate them here as they are
8
9
  # validated per each route (topic + consumer_group) because they can be overwritten,
9
- # so we validate all of that once all the routes are defined and ready
10
- class Config < Dry::Validation::Contract
11
- params do
12
- required(:client_id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
13
- required(:shutdown_timeout) { (int? & gt?(0)) }
14
- required(:consumer_mapper)
15
- required(:topic_mapper)
16
-
17
- optional(:backend).filled
10
+ # so we validate all of that once all the routes are defined and ready.
11
+ class Config < Base
12
+ configure do |config|
13
+ config.error_messages = YAML.safe_load(
14
+ File.read(
15
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
16
+ )
17
+ ).fetch('en').fetch('validations').fetch('config')
18
+ end
19
+
20
+ # License validity happens in the licenser. Here we do only the simple consistency checks
21
+ nested(:license) do
22
+ required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
23
+ required(:entity) { |val| val.is_a?(String) }
24
+ end
25
+
26
+ required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
27
+ required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
28
+ required(:consumer_mapper) { |val| !val.nil? }
29
+ required(:consumer_persistence) { |val| [true, false].include?(val) }
30
+ required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
31
+ required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
32
+ required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
33
+ required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
34
+ required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
35
+ required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
36
+
37
+ # We validate internals just to be sure, that they are present and working
38
+ nested(:internal) do
39
+ required(:status) { |val| !val.nil? }
40
+ required(:process) { |val| !val.nil? }
41
+
42
+ nested(:routing) do
43
+ required(:builder) { |val| !val.nil? }
44
+ required(:subscription_groups_builder) { |val| !val.nil? }
45
+ end
46
+
47
+ nested(:processing) do
48
+ required(:jobs_builder) { |val| !val.nil? }
49
+ required(:scheduler) { |val| !val.nil? }
50
+ required(:coordinator_class) { |val| !val.nil? }
51
+ required(:partitioner_class) { |val| !val.nil? }
52
+ end
53
+
54
+ nested(:active_job) do
55
+ required(:dispatcher) { |val| !val.nil? }
56
+ required(:job_options_contract) { |val| !val.nil? }
57
+ required(:consumer_class) { |val| !val.nil? }
58
+ end
59
+ end
60
+
61
+ virtual do |data, errors|
62
+ next unless errors.empty?
63
+
64
+ detected_errors = []
65
+
66
+ data.fetch(:kafka).each_key do |key|
67
+ next if key.is_a?(Symbol)
68
+
69
+ detected_errors << [[:kafka, key], :key_must_be_a_symbol]
70
+ end
71
+
72
+ detected_errors
73
+ end
74
+
75
+ virtual do |data, errors|
76
+ next unless errors.empty?
77
+
78
+ pause_timeout = data.fetch(:pause_timeout)
79
+ pause_max_timeout = data.fetch(:pause_max_timeout)
80
+
81
+ next if pause_timeout <= pause_max_timeout
82
+
83
+ [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
84
+ end
85
+
86
+ virtual do |data, errors|
87
+ next unless errors.empty?
88
+
89
+ shutdown_timeout = data.fetch(:shutdown_timeout)
90
+ max_wait_time = data.fetch(:max_wait_time)
91
+
92
+ next if max_wait_time < shutdown_timeout
93
+
94
+ [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
18
95
  end
19
96
  end
20
97
  end
@@ -3,208 +3,40 @@
3
3
  module Karafka
4
4
  module Contracts
5
5
  # Contract for single full route (consumer group + topics) validation.
6
- class ConsumerGroup < Dry::Validation::Contract
7
- config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
8
-
9
- # Valid uri schemas of Kafka broker url
10
- # The ||= is due to the behavior of require_all that resolves dependencies
11
- # but sometimes loads things twice
12
- URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
13
-
14
- # Available sasl scram mechanism of authentication (plus nil)
15
- SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
16
-
17
- # Internal contract for sub-validating topics schema
18
- TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
19
-
20
- private_constant :TOPIC_CONTRACT
21
-
22
- params do
23
- required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
24
- required(:topics).value(:array, :filled?)
25
- required(:seed_brokers).value(:array, :filled?)
26
- required(:session_timeout).filled { int? | float? }
27
- required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
28
- required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
29
- required(:pause_exponential_backoff).filled(:bool?)
30
- required(:offset_commit_interval) { int? | float? }
31
- required(:offset_commit_threshold).filled(:int?)
32
- required(:offset_retention_time).maybe(:integer)
33
- required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
34
- required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
35
- required(:assignment_strategy).value(:any)
36
- required(:connect_timeout).filled { (int? | float?) & gt?(0) }
37
- required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
38
- required(:socket_timeout).filled { (int? | float?) & gt?(0) }
39
- required(:min_bytes).filled(:int?, gt?: 0)
40
- required(:max_bytes).filled(:int?, gt?: 0)
41
- required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
42
- required(:batch_fetching).filled(:bool?)
43
-
44
- %i[
45
- ssl_ca_cert
46
- ssl_ca_cert_file_path
47
- ssl_client_cert
48
- ssl_client_cert_key
49
- ssl_client_cert_chain
50
- ssl_client_cert_key_password
51
- sasl_gssapi_principal
52
- sasl_gssapi_keytab
53
- sasl_plain_authzid
54
- sasl_plain_username
55
- sasl_plain_password
56
- sasl_scram_username
57
- sasl_scram_password
58
- ].each do |encryption_attribute|
59
- optional(encryption_attribute).maybe(:str?)
60
- end
61
-
62
- optional(:ssl_verify_hostname).maybe(:bool?)
63
- optional(:ssl_ca_certs_from_system).maybe(:bool?)
64
- optional(:sasl_over_ssl).maybe(:bool?)
65
- optional(:sasl_oauth_token_provider).value(:any)
66
-
67
- # It's not with other encryptions as it has some more rules
68
- optional(:sasl_scram_mechanism)
69
- .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
70
- end
71
-
72
- # Uri rule to check if uri is in a Karafka acceptable format
73
- rule(:seed_brokers) do
74
- if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
75
- key.failure(:invalid_broker_schema)
76
- end
77
- end
78
-
79
- rule(:topics) do
80
- if value.is_a?(Array)
81
- names = value.map { |topic| topic[:name] }
82
-
83
- key.failure(:topics_names_not_unique) if names.size != names.uniq.size
84
- end
85
- end
86
-
87
- rule(:topics) do
88
- if value.is_a?(Array)
89
- value.each_with_index do |topic, index|
90
- TOPIC_CONTRACT.call(topic).errors.each do |error|
91
- key([:topics, index, error.path[0]]).failure(error.text)
92
- end
93
- end
94
- end
95
- end
96
-
97
- rule(:assignment_strategy) do
98
- key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
99
- end
100
-
101
- rule(:ssl_client_cert, :ssl_client_cert_key) do
102
- if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
103
- key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
104
- end
105
- end
106
-
107
- rule(:ssl_client_cert, :ssl_client_cert_key) do
108
- if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
109
- key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
110
- end
6
+ class ConsumerGroup < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('consumer_group')
111
13
  end
112
14
 
113
- rule(:ssl_client_cert, :ssl_client_cert_chain) do
114
- if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
115
- key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
116
- end
117
- end
15
+ required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
16
+ required(:topics) { |topics| topics.is_a?(Array) && !topics.empty? }
118
17
 
119
- rule(:ssl_client_cert_chain, :ssl_client_cert_key) do
120
- if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
121
- key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
122
- end
123
- end
124
-
125
- rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
126
- if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
127
- key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
128
- end
129
- end
18
+ virtual do |data, errors|
19
+ next unless errors.empty?
130
20
 
131
- rule(:ssl_ca_cert) do
132
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
133
- end
21
+ names = data.fetch(:topics).map { |topic| topic[:name] }
134
22
 
135
- rule(:ssl_client_cert) do
136
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
137
- end
23
+ next if names.size == names.uniq.size
138
24
 
139
- rule(:ssl_ca_cert_file_path) do
140
- if value
141
- if File.exist?(value)
142
- key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
143
- else
144
- key.failure(:does_not_exist)
145
- end
146
- end
25
+ [[%i[topics], :names_not_unique]]
147
26
  end
148
27
 
149
- rule(:ssl_client_cert_key) do
150
- key.failure(:invalid_private_key) if value && !valid_private_key?(value)
151
- end
28
+ virtual do |data, errors|
29
+ next unless errors.empty?
152
30
 
153
- rule(:ssl_client_cert_chain) do
154
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
155
- end
31
+ fetched_errors = []
156
32
 
157
- rule(:sasl_oauth_token_provider) do
158
- key.failure(:does_not_respond_to_token) if value && !value.respond_to?(:token)
159
- end
160
-
161
- rule(:max_wait_time, :socket_timeout) do
162
- max_wait_time = values[:max_wait_time]
163
- socket_timeout = values[:socket_timeout]
164
-
165
- if socket_timeout.is_a?(Numeric) &&
166
- max_wait_time.is_a?(Numeric) &&
167
- max_wait_time > socket_timeout
168
-
169
- key(:max_wait_time).failure(:max_wait_time_limit)
170
- end
171
- end
172
-
173
- rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
174
- if values[:pause_exponential_backoff]
175
- if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
176
- key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
33
+ data.fetch(:topics).each do |topic|
34
+ ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
35
+ fetched_errors << [[topic, key].flatten, value]
177
36
  end
178
37
  end
179
- end
180
-
181
- private
182
-
183
- # @param value [String] potential RSA key value
184
- # @return [Boolean] is the given string a valid RSA key
185
- def valid_private_key?(value)
186
- OpenSSL::PKey.read(value)
187
- true
188
- rescue OpenSSL::PKey::PKeyError
189
- false
190
- end
191
-
192
- # @param value [String] potential X509 cert value
193
- # @return [Boolean] is the given string a valid X509 cert
194
- def valid_certificate?(value)
195
- OpenSSL::X509::Certificate.new(value)
196
- true
197
- rescue OpenSSL::X509::CertificateError
198
- false
199
- end
200
38
 
201
- # @param value [String] potential kafka uri
202
- # @return [Boolean] true if it is a kafka uri, otherwise false
203
- def kafka_uri?(value)
204
- uri = URI.parse(value)
205
- URI_SCHEMES.include?(uri.scheme) && uri.port
206
- rescue URI::InvalidURIError
207
- false
39
+ fetched_errors
208
40
  end
209
41
  end
210
42
  end
@@ -2,17 +2,40 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Consumer group topic validation rules
6
- class ConsumerGroupTopic < Dry::Validation::Contract
7
- params do
8
- required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
9
- required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
10
- required(:backend).filled(included_in?: %i[inline sidekiq])
11
- required(:consumer).filled
12
- required(:deserializer).filled
13
- required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
14
- required(:start_from_beginning).filled(:bool?)
15
- required(:batch_consuming).filled(:bool?)
5
+ # Consumer group topic validation rules.
6
+ class ConsumerGroupTopic < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('consumer_group_topic')
13
+ end
14
+
15
+ required(:consumer) { |consumer_group| !consumer_group.nil? }
16
+ required(:deserializer) { |deserializer| !deserializer.nil? }
17
+ required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
18
+ required(:kafka) { |kafka| kafka.is_a?(Hash) && !kafka.empty? }
19
+ required(:max_messages) { |mm| mm.is_a?(Integer) && mm >= 1 }
20
+ required(:initial_offset) { |io| %w[earliest latest].include?(io) }
21
+ required(:max_wait_time) { |mwt| mwt.is_a?(Integer) && mwt >= 10 }
22
+ required(:manual_offset_management) { |mmm| [true, false].include?(mmm) }
23
+ required(:name) { |name| name.is_a?(String) && Contracts::TOPIC_REGEXP.match?(name) }
24
+
25
+ virtual do |data, errors|
26
+ next unless errors.empty?
27
+
28
+ value = data.fetch(:kafka)
29
+
30
+ begin
31
+ # This will trigger rdkafka validations that we catch and re-map the info and use dry
32
+ # compatible format
33
+ Rdkafka::Config.new(value).send(:native_config)
34
+
35
+ nil
36
+ rescue Rdkafka::Config::ConfigError => e
37
+ [[%w[kafka], e.message]]
38
+ end
16
39
  end
17
40
  end
18
41
  end
@@ -2,29 +2,30 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Contract for validating correctness of the server cli command options
6
- # We validate some basics + the list of consumer_groups on which we want to use, to make
7
- # sure that all of them are defined, plus that a pidfile does not exist
8
- class ServerCliOptions < Dry::Validation::Contract
9
- config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
10
-
11
- params do
12
- optional(:pid).filled(:str?)
13
- optional(:daemon).filled(:bool?)
14
- optional(:consumer_groups).value(:array, :filled?)
5
+ # Contract for validating correctness of the server cli command options.
6
+ class ServerCliOptions < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('server_cli_options')
15
13
  end
16
14
 
17
- rule(:pid) do
18
- key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
19
- end
15
+ optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
16
+
17
+ virtual do |data, errors|
18
+ next unless errors.empty?
19
+ next unless data.key?(:consumer_groups)
20
+
21
+ value = data.fetch(:consumer_groups)
20
22
 
21
- rule(:consumer_groups) do
22
23
  # If there were no consumer_groups declared in the server cli, it means that we will
23
24
  # run all of them and no need to validate them here at all
24
- if !value.nil? &&
25
- !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
26
- key(:consumer_groups).failure(:consumer_groups_inclusion)
27
- end
25
+ next if value.nil?
26
+ next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
27
+
28
+ [[%i[consumer_groups], :consumer_groups_inclusion]]
28
29
  end
29
30
  end
30
31
  end
@@ -5,6 +5,6 @@ module Karafka
5
5
  module Contracts
6
6
  # Regexp for validating format of groups and topics
7
7
  # @note It is not nested inside of the contracts, as it is used by couple of them
8
- TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
8
+ TOPIC_REGEXP = /\A(\w|-|\.)+\z/
9
9
  end
10
10
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Env management class to get and set environment for Karafka
5
+ class Env < String
6
+ # Keys where we look for environment details for Karafka
7
+ LOOKUP_ENV_KEYS = %w[
8
+ KARAFKA_ENV
9
+ RACK_ENV
10
+ RAILS_ENV
11
+ ].freeze
12
+
13
+ # Default fallback env
14
+ DEFAULT_ENV = 'development'
15
+
16
+ private_constant :LOOKUP_ENV_KEYS, :DEFAULT_ENV
17
+
18
+ # @return [Karafka::Env] env object
19
+ # @note Will load appropriate environment automatically
20
+ def initialize
21
+ super('')
22
+
23
+ LOOKUP_ENV_KEYS
24
+ .map { |key| ENV[key] }
25
+ .compact
26
+ .first
27
+ .then { |env| env || DEFAULT_ENV }
28
+ .then { |env| replace(env) }
29
+ end
30
+
31
+ # @param method_name [String] method name
32
+ # @param include_private [Boolean] should we include private methods as well
33
+ # @return [Boolean] true if we respond to a given missing method, otherwise false
34
+ def respond_to_missing?(method_name, include_private = false)
35
+ (method_name[-1] == '?') || super
36
+ end
37
+
38
+ # Reacts to missing methods, from which some might be the env checks.
39
+ # If the method ends with '?' we assume, that it is an env check
40
+ # @param method_name [String] method name for missing or env name with question mark
41
+ # @param arguments [Array] any arguments that we pass to the method
42
+ def method_missing(method_name, *arguments)
43
+ method_name[-1] == '?' ? self == method_name[0..-2] : super
44
+ end
45
+ end
46
+ end
@@ -6,12 +6,6 @@ module Karafka
6
6
  # Base class for all the Karafka internal errors
7
7
  BaseError = Class.new(StandardError)
8
8
 
9
- # Should be raised when we have that that we cannot serialize
10
- SerializationError = Class.new(BaseError)
11
-
12
- # Should be raised when we tried to deserialize incoming data but we failed
13
- DeserializationError = Class.new(BaseError)
14
-
15
9
  # Raised when router receives topic name which does not correspond with any routes
16
10
  # This can only happen in a case when:
17
11
  # - you've received a message and we cannot match it with a consumer
@@ -24,28 +18,34 @@ module Karafka
24
18
  # @see https://github.com/karafka/karafka/issues/135
25
19
  NonMatchingRouteError = Class.new(BaseError)
26
20
 
27
- # Raised when we don't use or use responder not in the way it expected to based on the
28
- # topics usage definitions
29
- InvalidResponderUsageError = Class.new(BaseError)
30
-
31
- # Raised when options that we provide to the responder to respond aren't what the contract
32
- # requires
33
- InvalidResponderMessageOptionsError = Class.new(BaseError)
34
-
35
21
  # Raised when configuration doesn't match with validation contract
36
22
  InvalidConfigurationError = Class.new(BaseError)
37
23
 
38
24
  # Raised when we try to use Karafka CLI commands (except install) without a boot file
39
25
  MissingBootFileError = Class.new(BaseError)
40
26
 
41
- # Raised when we want to read a persisted thread messages consumer but it is unavailable
42
- # This should never happen and if it does, please contact us
43
- MissingClientError = Class.new(BaseError)
44
-
45
- # Raised when want to hook up to an event that is not registered and supported
46
- UnregisteredMonitorEventError = Class.new(BaseError)
47
-
48
27
  # Raised when we've waited enough for shutting down a non-responsive process
49
28
  ForcefulShutdownError = Class.new(BaseError)
29
+
30
+ # Raised when the jobs queue receives a job that should not be received as it would cause
31
+ # the processing to go out of sync. We should never process in parallel data from the same
32
+ # topic partition (unless virtual partitions apply)
33
+ JobsQueueSynchronizationError = Class.new(BaseError)
34
+
35
+ # Raised when given topic is not found while expected
36
+ TopicNotFoundError = Class.new(BaseError)
37
+
38
+ # This should never happen. Please open an issue if it does.
39
+ UnsupportedCaseError = Class.new(BaseError)
40
+
41
+ # Raised when the license token is not valid
42
+ InvalidLicenseTokenError = Class.new(BaseError)
43
+
44
+ # Used to instrument this error into the error notifications
45
+ # We do not raise it so we won't crash deployed systems
46
+ ExpiredLicenseTokenError = Class.new(BaseError)
47
+
48
+ # This should never happen. Please open an issue if it does.
49
+ InvalidCoordinatorState = Class.new(BaseError)
50
50
  end
51
51
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Helpers
5
+ # Allows a given class to run async in a separate thread. Provides also few methods we may
6
+ # want to use to control the underlying thread
7
+ #
8
+ # @note Thread running code needs to manage it's own exceptions. If they leak out, they will
9
+ # abort thread on exception.
10
+ module Async
11
+ class << self
12
+ # Adds forwardable to redirect thread-based control methods to the underlying thread that
13
+ # runs the async operations
14
+ #
15
+ # @param base [Class] class we're including this module in
16
+ def included(base)
17
+ base.extend ::Forwardable
18
+
19
+ base.def_delegators :@thread, :join, :terminate, :alive?
20
+ end
21
+ end
22
+
23
+ # Runs the `#call` method in a new thread
24
+ def async_call
25
+ @thread = Thread.new do
26
+ Thread.current.abort_on_exception = true
27
+
28
+ call
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Helpers
5
+ # Simple wrapper for adding colors to strings
6
+ module Colorize
7
+ # @param string [String] string we want to have in green
8
+ # @return [String] green string
9
+ def green(string)
10
+ "\033[0;32m#{string}\033[0m"
11
+ end
12
+
13
+ # @param string [String] string we want to have in red
14
+ # @return [String] red string
15
+ def red(string)
16
+ "\033[0;31m#{string}\033[0m"
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Karafka
4
- # Module containing classes and methods that provide some additional functionalities
4
+ # Module containing classes and methods that provide some additional helper functionalities.
5
5
  module Helpers
6
6
  # @note Taken from http://stackoverflow.com/questions/6407141
7
- # Multidelegator is used to delegate calls to multiple targets
7
+ # Multidelegator is used to delegate calls to multiple targets.
8
8
  class MultiDelegator
9
9
  # @param targets to which we want to delegate methods
10
10
  def initialize(*targets)
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Instrumentation
5
+ # Callbacks used to transport things from rdkafka
6
+ module Callbacks
7
+ # Callback that kicks in when consumer error occurs and is published in a background thread
8
+ class Error
9
+ # @param subscription_group_id [String] id of the current subscription group instance
10
+ # @param consumer_group_id [String] id of the current consumer group
11
+ # @param client_name [String] rdkafka client name
12
+ # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
13
+ def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
14
+ @subscription_group_id = subscription_group_id
15
+ @consumer_group_id = consumer_group_id
16
+ @client_name = client_name
17
+ @monitor = monitor
18
+ end
19
+
20
+ # Runs the instrumentation monitor with error
21
+ # @param client_name [String] rdkafka client name
22
+ # @param error [Rdkafka::Error] error that occurred
23
+ # @note It will only instrument on errors of the client of our consumer
24
+ def call(client_name, error)
25
+ # Emit only errors related to our client
26
+ # Same as with statistics (mor explanation there)
27
+ return unless @client_name == client_name
28
+
29
+ @monitor.instrument(
30
+ 'error.occurred',
31
+ subscription_group_id: @subscription_group_id,
32
+ consumer_group_id: @consumer_group_id,
33
+ type: 'librdkafka.error',
34
+ error: error
35
+ )
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end