karafka 1.4.13 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (170) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +3 -3
  3. data/.github/workflows/ci.yml +85 -30
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +268 -7
  6. data/CONTRIBUTING.md +10 -19
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +44 -87
  9. data/LICENSE +17 -0
  10. data/LICENSE-COMM +89 -0
  11. data/LICENSE-LGPL +165 -0
  12. data/README.md +44 -48
  13. data/bin/benchmarks +85 -0
  14. data/bin/create_token +22 -0
  15. data/bin/integrations +237 -0
  16. data/bin/karafka +4 -0
  17. data/bin/scenario +29 -0
  18. data/bin/stress_many +13 -0
  19. data/bin/stress_one +13 -0
  20. data/bin/wait_for_kafka +20 -0
  21. data/certs/karafka-pro.pem +11 -0
  22. data/config/errors.yml +55 -40
  23. data/docker-compose.yml +39 -3
  24. data/karafka.gemspec +11 -17
  25. data/lib/active_job/karafka.rb +21 -0
  26. data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
  27. data/lib/karafka/active_job/consumer.rb +26 -0
  28. data/lib/karafka/active_job/dispatcher.rb +38 -0
  29. data/lib/karafka/active_job/job_extensions.rb +34 -0
  30. data/lib/karafka/active_job/job_options_contract.rb +21 -0
  31. data/lib/karafka/active_job/routing/extensions.rb +31 -0
  32. data/lib/karafka/app.rb +15 -20
  33. data/lib/karafka/base_consumer.rb +181 -31
  34. data/lib/karafka/cli/base.rb +4 -4
  35. data/lib/karafka/cli/info.rb +43 -9
  36. data/lib/karafka/cli/install.rb +19 -10
  37. data/lib/karafka/cli/server.rb +17 -42
  38. data/lib/karafka/cli.rb +4 -11
  39. data/lib/karafka/connection/client.rb +385 -90
  40. data/lib/karafka/connection/listener.rb +246 -38
  41. data/lib/karafka/connection/listeners_batch.rb +24 -0
  42. data/lib/karafka/connection/messages_buffer.rb +84 -0
  43. data/lib/karafka/connection/pauses_manager.rb +46 -0
  44. data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
  45. data/lib/karafka/connection/rebalance_manager.rb +78 -0
  46. data/lib/karafka/contracts/base.rb +17 -0
  47. data/lib/karafka/contracts/config.rb +88 -11
  48. data/lib/karafka/contracts/consumer_group.rb +21 -189
  49. data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
  50. data/lib/karafka/contracts/server_cli_options.rb +19 -18
  51. data/lib/karafka/contracts.rb +1 -1
  52. data/lib/karafka/env.rb +46 -0
  53. data/lib/karafka/errors.rb +21 -21
  54. data/lib/karafka/helpers/async.rb +33 -0
  55. data/lib/karafka/helpers/colorize.rb +20 -0
  56. data/lib/karafka/helpers/multi_delegator.rb +2 -2
  57. data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
  58. data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
  59. data/lib/karafka/instrumentation/logger_listener.rb +164 -0
  60. data/lib/karafka/instrumentation/monitor.rb +13 -61
  61. data/lib/karafka/instrumentation/notifications.rb +52 -0
  62. data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
  63. data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
  64. data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
  65. data/lib/karafka/instrumentation.rb +21 -0
  66. data/lib/karafka/licenser.rb +75 -0
  67. data/lib/karafka/messages/batch_metadata.rb +45 -0
  68. data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
  69. data/lib/karafka/messages/builders/message.rb +39 -0
  70. data/lib/karafka/messages/builders/messages.rb +32 -0
  71. data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
  72. data/lib/karafka/messages/messages.rb +64 -0
  73. data/lib/karafka/{params → messages}/metadata.rb +4 -6
  74. data/lib/karafka/messages/seek.rb +9 -0
  75. data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
  76. data/lib/karafka/pro/active_job/consumer.rb +46 -0
  77. data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
  78. data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
  79. data/lib/karafka/pro/base_consumer.rb +82 -0
  80. data/lib/karafka/pro/contracts/base.rb +21 -0
  81. data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
  82. data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
  83. data/lib/karafka/pro/loader.rb +76 -0
  84. data/lib/karafka/pro/performance_tracker.rb +80 -0
  85. data/lib/karafka/pro/processing/coordinator.rb +72 -0
  86. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
  87. data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
  88. data/lib/karafka/pro/processing/partitioner.rb +60 -0
  89. data/lib/karafka/pro/processing/scheduler.rb +56 -0
  90. data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
  91. data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
  92. data/lib/karafka/pro.rb +13 -0
  93. data/lib/karafka/process.rb +1 -0
  94. data/lib/karafka/processing/coordinator.rb +88 -0
  95. data/lib/karafka/processing/coordinators_buffer.rb +54 -0
  96. data/lib/karafka/processing/executor.rb +118 -0
  97. data/lib/karafka/processing/executors_buffer.rb +88 -0
  98. data/lib/karafka/processing/jobs/base.rb +51 -0
  99. data/lib/karafka/processing/jobs/consume.rb +42 -0
  100. data/lib/karafka/processing/jobs/revoked.rb +22 -0
  101. data/lib/karafka/processing/jobs/shutdown.rb +23 -0
  102. data/lib/karafka/processing/jobs_builder.rb +29 -0
  103. data/lib/karafka/processing/jobs_queue.rb +144 -0
  104. data/lib/karafka/processing/partitioner.rb +22 -0
  105. data/lib/karafka/processing/result.rb +29 -0
  106. data/lib/karafka/processing/scheduler.rb +22 -0
  107. data/lib/karafka/processing/worker.rb +88 -0
  108. data/lib/karafka/processing/workers_batch.rb +27 -0
  109. data/lib/karafka/railtie.rb +113 -0
  110. data/lib/karafka/routing/builder.rb +15 -24
  111. data/lib/karafka/routing/consumer_group.rb +11 -19
  112. data/lib/karafka/routing/consumer_mapper.rb +1 -2
  113. data/lib/karafka/routing/router.rb +1 -1
  114. data/lib/karafka/routing/subscription_group.rb +53 -0
  115. data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
  116. data/lib/karafka/routing/topic.rb +61 -24
  117. data/lib/karafka/routing/topics.rb +38 -0
  118. data/lib/karafka/runner.rb +51 -0
  119. data/lib/karafka/serialization/json/deserializer.rb +6 -15
  120. data/lib/karafka/server.rb +67 -26
  121. data/lib/karafka/setup/config.rb +147 -175
  122. data/lib/karafka/status.rb +14 -5
  123. data/lib/karafka/templates/example_consumer.rb.erb +16 -0
  124. data/lib/karafka/templates/karafka.rb.erb +15 -51
  125. data/lib/karafka/time_trackers/base.rb +19 -0
  126. data/lib/karafka/time_trackers/pause.rb +92 -0
  127. data/lib/karafka/time_trackers/poll.rb +65 -0
  128. data/lib/karafka/version.rb +1 -1
  129. data/lib/karafka.rb +38 -17
  130. data.tar.gz.sig +0 -0
  131. metadata +118 -120
  132. metadata.gz.sig +0 -0
  133. data/MIT-LICENCE +0 -18
  134. data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
  135. data/lib/karafka/attributes_map.rb +0 -63
  136. data/lib/karafka/backends/inline.rb +0 -16
  137. data/lib/karafka/base_responder.rb +0 -226
  138. data/lib/karafka/cli/flow.rb +0 -48
  139. data/lib/karafka/cli/missingno.rb +0 -19
  140. data/lib/karafka/code_reloader.rb +0 -67
  141. data/lib/karafka/connection/api_adapter.rb +0 -158
  142. data/lib/karafka/connection/batch_delegator.rb +0 -55
  143. data/lib/karafka/connection/builder.rb +0 -23
  144. data/lib/karafka/connection/message_delegator.rb +0 -36
  145. data/lib/karafka/consumers/batch_metadata.rb +0 -10
  146. data/lib/karafka/consumers/callbacks.rb +0 -71
  147. data/lib/karafka/consumers/includer.rb +0 -64
  148. data/lib/karafka/consumers/responders.rb +0 -24
  149. data/lib/karafka/consumers/single_params.rb +0 -15
  150. data/lib/karafka/contracts/responder_usage.rb +0 -54
  151. data/lib/karafka/fetcher.rb +0 -42
  152. data/lib/karafka/helpers/class_matcher.rb +0 -88
  153. data/lib/karafka/helpers/config_retriever.rb +0 -46
  154. data/lib/karafka/helpers/inflector.rb +0 -26
  155. data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
  156. data/lib/karafka/params/batch_metadata.rb +0 -26
  157. data/lib/karafka/params/builders/batch_metadata.rb +0 -30
  158. data/lib/karafka/params/builders/params.rb +0 -38
  159. data/lib/karafka/params/builders/params_batch.rb +0 -25
  160. data/lib/karafka/params/params_batch.rb +0 -60
  161. data/lib/karafka/patches/ruby_kafka.rb +0 -47
  162. data/lib/karafka/persistence/client.rb +0 -29
  163. data/lib/karafka/persistence/consumers.rb +0 -45
  164. data/lib/karafka/persistence/topics.rb +0 -48
  165. data/lib/karafka/responders/builder.rb +0 -36
  166. data/lib/karafka/responders/topic.rb +0 -55
  167. data/lib/karafka/routing/topic_mapper.rb +0 -53
  168. data/lib/karafka/serialization/json/serializer.rb +0 -31
  169. data/lib/karafka/setup/configurators/water_drop.rb +0 -36
  170. data/lib/karafka/templates/application_responder.rb.erb +0 -11
@@ -2,19 +2,96 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Contract with validation rules for Karafka configuration details
5
+ # Contract with validation rules for Karafka configuration details.
6
+ #
6
7
  # @note There are many more configuration options inside of the
7
- # Karafka::Setup::Config model, but we don't validate them here as they are
8
+ # `Karafka::Setup::Config` model, but we don't validate them here as they are
8
9
  # validated per each route (topic + consumer_group) because they can be overwritten,
9
- # so we validate all of that once all the routes are defined and ready
10
- class Config < Dry::Validation::Contract
11
- params do
12
- required(:client_id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
13
- required(:shutdown_timeout) { (int? & gt?(0)) }
14
- required(:consumer_mapper)
15
- required(:topic_mapper)
16
-
17
- optional(:backend).filled
10
+ # so we validate all of that once all the routes are defined and ready.
11
+ class Config < Base
12
+ configure do |config|
13
+ config.error_messages = YAML.safe_load(
14
+ File.read(
15
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
16
+ )
17
+ ).fetch('en').fetch('validations').fetch('config')
18
+ end
19
+
20
+ # License validity happens in the licenser. Here we do only the simple consistency checks
21
+ nested(:license) do
22
+ required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
23
+ required(:entity) { |val| val.is_a?(String) }
24
+ end
25
+
26
+ required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
27
+ required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
28
+ required(:consumer_mapper) { |val| !val.nil? }
29
+ required(:consumer_persistence) { |val| [true, false].include?(val) }
30
+ required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
31
+ required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
32
+ required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
33
+ required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
34
+ required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
35
+ required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
36
+
37
+ # We validate internals just to be sure, that they are present and working
38
+ nested(:internal) do
39
+ required(:status) { |val| !val.nil? }
40
+ required(:process) { |val| !val.nil? }
41
+
42
+ nested(:routing) do
43
+ required(:builder) { |val| !val.nil? }
44
+ required(:subscription_groups_builder) { |val| !val.nil? }
45
+ end
46
+
47
+ nested(:processing) do
48
+ required(:jobs_builder) { |val| !val.nil? }
49
+ required(:scheduler) { |val| !val.nil? }
50
+ required(:coordinator_class) { |val| !val.nil? }
51
+ required(:partitioner_class) { |val| !val.nil? }
52
+ end
53
+
54
+ nested(:active_job) do
55
+ required(:dispatcher) { |val| !val.nil? }
56
+ required(:job_options_contract) { |val| !val.nil? }
57
+ required(:consumer_class) { |val| !val.nil? }
58
+ end
59
+ end
60
+
61
+ virtual do |data, errors|
62
+ next unless errors.empty?
63
+
64
+ detected_errors = []
65
+
66
+ data.fetch(:kafka).each_key do |key|
67
+ next if key.is_a?(Symbol)
68
+
69
+ detected_errors << [[:kafka, key], :key_must_be_a_symbol]
70
+ end
71
+
72
+ detected_errors
73
+ end
74
+
75
+ virtual do |data, errors|
76
+ next unless errors.empty?
77
+
78
+ pause_timeout = data.fetch(:pause_timeout)
79
+ pause_max_timeout = data.fetch(:pause_max_timeout)
80
+
81
+ next if pause_timeout <= pause_max_timeout
82
+
83
+ [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
84
+ end
85
+
86
+ virtual do |data, errors|
87
+ next unless errors.empty?
88
+
89
+ shutdown_timeout = data.fetch(:shutdown_timeout)
90
+ max_wait_time = data.fetch(:max_wait_time)
91
+
92
+ next if max_wait_time < shutdown_timeout
93
+
94
+ [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
18
95
  end
19
96
  end
20
97
  end
@@ -3,208 +3,40 @@
3
3
  module Karafka
4
4
  module Contracts
5
5
  # Contract for single full route (consumer group + topics) validation.
6
- class ConsumerGroup < Dry::Validation::Contract
7
- config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
8
-
9
- # Valid uri schemas of Kafka broker url
10
- # The ||= is due to the behavior of require_all that resolves dependencies
11
- # but sometimes loads things twice
12
- URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
13
-
14
- # Available sasl scram mechanism of authentication (plus nil)
15
- SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
16
-
17
- # Internal contract for sub-validating topics schema
18
- TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
19
-
20
- private_constant :TOPIC_CONTRACT
21
-
22
- params do
23
- required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
24
- required(:topics).value(:array, :filled?)
25
- required(:seed_brokers).value(:array, :filled?)
26
- required(:session_timeout).filled { int? | float? }
27
- required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
28
- required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
29
- required(:pause_exponential_backoff).filled(:bool?)
30
- required(:offset_commit_interval) { int? | float? }
31
- required(:offset_commit_threshold).filled(:int?)
32
- required(:offset_retention_time).maybe(:integer)
33
- required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
34
- required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
35
- required(:assignment_strategy).value(:any)
36
- required(:connect_timeout).filled { (int? | float?) & gt?(0) }
37
- required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
38
- required(:socket_timeout).filled { (int? | float?) & gt?(0) }
39
- required(:min_bytes).filled(:int?, gt?: 0)
40
- required(:max_bytes).filled(:int?, gt?: 0)
41
- required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
42
- required(:batch_fetching).filled(:bool?)
43
-
44
- %i[
45
- ssl_ca_cert
46
- ssl_ca_cert_file_path
47
- ssl_client_cert
48
- ssl_client_cert_key
49
- ssl_client_cert_chain
50
- ssl_client_cert_key_password
51
- sasl_gssapi_principal
52
- sasl_gssapi_keytab
53
- sasl_plain_authzid
54
- sasl_plain_username
55
- sasl_plain_password
56
- sasl_scram_username
57
- sasl_scram_password
58
- ].each do |encryption_attribute|
59
- optional(encryption_attribute).maybe(:str?)
60
- end
61
-
62
- optional(:ssl_verify_hostname).maybe(:bool?)
63
- optional(:ssl_ca_certs_from_system).maybe(:bool?)
64
- optional(:sasl_over_ssl).maybe(:bool?)
65
- optional(:sasl_oauth_token_provider).value(:any)
66
-
67
- # It's not with other encryptions as it has some more rules
68
- optional(:sasl_scram_mechanism)
69
- .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
70
- end
71
-
72
- # Uri rule to check if uri is in a Karafka acceptable format
73
- rule(:seed_brokers) do
74
- if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
75
- key.failure(:invalid_broker_schema)
76
- end
77
- end
78
-
79
- rule(:topics) do
80
- if value.is_a?(Array)
81
- names = value.map { |topic| topic[:name] }
82
-
83
- key.failure(:topics_names_not_unique) if names.size != names.uniq.size
84
- end
85
- end
86
-
87
- rule(:topics) do
88
- if value.is_a?(Array)
89
- value.each_with_index do |topic, index|
90
- TOPIC_CONTRACT.call(topic).errors.each do |error|
91
- key([:topics, index, error.path[0]]).failure(error.text)
92
- end
93
- end
94
- end
95
- end
96
-
97
- rule(:assignment_strategy) do
98
- key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
99
- end
100
-
101
- rule(:ssl_client_cert, :ssl_client_cert_key) do
102
- if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
103
- key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
104
- end
105
- end
106
-
107
- rule(:ssl_client_cert, :ssl_client_cert_key) do
108
- if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
109
- key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
110
- end
6
+ class ConsumerGroup < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('consumer_group')
111
13
  end
112
14
 
113
- rule(:ssl_client_cert, :ssl_client_cert_chain) do
114
- if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
115
- key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
116
- end
117
- end
15
+ required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
16
+ required(:topics) { |topics| topics.is_a?(Array) && !topics.empty? }
118
17
 
119
- rule(:ssl_client_cert_chain, :ssl_client_cert_key) do
120
- if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
121
- key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
122
- end
123
- end
124
-
125
- rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
126
- if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
127
- key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
128
- end
129
- end
18
+ virtual do |data, errors|
19
+ next unless errors.empty?
130
20
 
131
- rule(:ssl_ca_cert) do
132
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
133
- end
21
+ names = data.fetch(:topics).map { |topic| topic[:name] }
134
22
 
135
- rule(:ssl_client_cert) do
136
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
137
- end
23
+ next if names.size == names.uniq.size
138
24
 
139
- rule(:ssl_ca_cert_file_path) do
140
- if value
141
- if File.exist?(value)
142
- key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
143
- else
144
- key.failure(:does_not_exist)
145
- end
146
- end
25
+ [[%i[topics], :names_not_unique]]
147
26
  end
148
27
 
149
- rule(:ssl_client_cert_key) do
150
- key.failure(:invalid_private_key) if value && !valid_private_key?(value)
151
- end
28
+ virtual do |data, errors|
29
+ next unless errors.empty?
152
30
 
153
- rule(:ssl_client_cert_chain) do
154
- key.failure(:invalid_certificate) if value && !valid_certificate?(value)
155
- end
31
+ fetched_errors = []
156
32
 
157
- rule(:sasl_oauth_token_provider) do
158
- key.failure(:does_not_respond_to_token) if value && !value.respond_to?(:token)
159
- end
160
-
161
- rule(:max_wait_time, :socket_timeout) do
162
- max_wait_time = values[:max_wait_time]
163
- socket_timeout = values[:socket_timeout]
164
-
165
- if socket_timeout.is_a?(Numeric) &&
166
- max_wait_time.is_a?(Numeric) &&
167
- max_wait_time > socket_timeout
168
-
169
- key(:max_wait_time).failure(:max_wait_time_limit)
170
- end
171
- end
172
-
173
- rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
174
- if values[:pause_exponential_backoff]
175
- if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
176
- key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
33
+ data.fetch(:topics).each do |topic|
34
+ ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
35
+ fetched_errors << [[topic, key].flatten, value]
177
36
  end
178
37
  end
179
- end
180
-
181
- private
182
-
183
- # @param value [String] potential RSA key value
184
- # @return [Boolean] is the given string a valid RSA key
185
- def valid_private_key?(value)
186
- OpenSSL::PKey.read(value)
187
- true
188
- rescue OpenSSL::PKey::PKeyError
189
- false
190
- end
191
-
192
- # @param value [String] potential X509 cert value
193
- # @return [Boolean] is the given string a valid X509 cert
194
- def valid_certificate?(value)
195
- OpenSSL::X509::Certificate.new(value)
196
- true
197
- rescue OpenSSL::X509::CertificateError
198
- false
199
- end
200
38
 
201
- # @param value [String] potential kafka uri
202
- # @return [Boolean] true if it is a kafka uri, otherwise false
203
- def kafka_uri?(value)
204
- uri = URI.parse(value)
205
- URI_SCHEMES.include?(uri.scheme) && uri.port
206
- rescue URI::InvalidURIError
207
- false
39
+ fetched_errors
208
40
  end
209
41
  end
210
42
  end
@@ -2,17 +2,40 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Consumer group topic validation rules
6
- class ConsumerGroupTopic < Dry::Validation::Contract
7
- params do
8
- required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
9
- required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
10
- required(:backend).filled(included_in?: %i[inline sidekiq])
11
- required(:consumer).filled
12
- required(:deserializer).filled
13
- required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
14
- required(:start_from_beginning).filled(:bool?)
15
- required(:batch_consuming).filled(:bool?)
5
+ # Consumer group topic validation rules.
6
+ class ConsumerGroupTopic < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('consumer_group_topic')
13
+ end
14
+
15
+ required(:consumer) { |consumer_group| !consumer_group.nil? }
16
+ required(:deserializer) { |deserializer| !deserializer.nil? }
17
+ required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
18
+ required(:kafka) { |kafka| kafka.is_a?(Hash) && !kafka.empty? }
19
+ required(:max_messages) { |mm| mm.is_a?(Integer) && mm >= 1 }
20
+ required(:initial_offset) { |io| %w[earliest latest].include?(io) }
21
+ required(:max_wait_time) { |mwt| mwt.is_a?(Integer) && mwt >= 10 }
22
+ required(:manual_offset_management) { |mmm| [true, false].include?(mmm) }
23
+ required(:name) { |name| name.is_a?(String) && Contracts::TOPIC_REGEXP.match?(name) }
24
+
25
+ virtual do |data, errors|
26
+ next unless errors.empty?
27
+
28
+ value = data.fetch(:kafka)
29
+
30
+ begin
31
+ # This will trigger rdkafka validations that we catch and re-map the info and use dry
32
+ # compatible format
33
+ Rdkafka::Config.new(value).send(:native_config)
34
+
35
+ nil
36
+ rescue Rdkafka::Config::ConfigError => e
37
+ [[%w[kafka], e.message]]
38
+ end
16
39
  end
17
40
  end
18
41
  end
@@ -2,29 +2,30 @@
2
2
 
3
3
  module Karafka
4
4
  module Contracts
5
- # Contract for validating correctness of the server cli command options
6
- # We validate some basics + the list of consumer_groups on which we want to use, to make
7
- # sure that all of them are defined, plus that a pidfile does not exist
8
- class ServerCliOptions < Dry::Validation::Contract
9
- config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
10
-
11
- params do
12
- optional(:pid).filled(:str?)
13
- optional(:daemon).filled(:bool?)
14
- optional(:consumer_groups).value(:array, :filled?)
5
+ # Contract for validating correctness of the server cli command options.
6
+ class ServerCliOptions < Base
7
+ configure do |config|
8
+ config.error_messages = YAML.safe_load(
9
+ File.read(
10
+ File.join(Karafka.gem_root, 'config', 'errors.yml')
11
+ )
12
+ ).fetch('en').fetch('validations').fetch('server_cli_options')
15
13
  end
16
14
 
17
- rule(:pid) do
18
- key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
19
- end
15
+ optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
16
+
17
+ virtual do |data, errors|
18
+ next unless errors.empty?
19
+ next unless data.key?(:consumer_groups)
20
+
21
+ value = data.fetch(:consumer_groups)
20
22
 
21
- rule(:consumer_groups) do
22
23
  # If there were no consumer_groups declared in the server cli, it means that we will
23
24
  # run all of them and no need to validate them here at all
24
- if !value.nil? &&
25
- !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
26
- key(:consumer_groups).failure(:consumer_groups_inclusion)
27
- end
25
+ next if value.nil?
26
+ next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
27
+
28
+ [[%i[consumer_groups], :consumer_groups_inclusion]]
28
29
  end
29
30
  end
30
31
  end
@@ -5,6 +5,6 @@ module Karafka
5
5
  module Contracts
6
6
  # Regexp for validating format of groups and topics
7
7
  # @note It is not nested inside of the contracts, as it is used by couple of them
8
- TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
8
+ TOPIC_REGEXP = /\A(\w|-|\.)+\z/
9
9
  end
10
10
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Env management class to get and set environment for Karafka
5
+ class Env < String
6
+ # Keys where we look for environment details for Karafka
7
+ LOOKUP_ENV_KEYS = %w[
8
+ KARAFKA_ENV
9
+ RACK_ENV
10
+ RAILS_ENV
11
+ ].freeze
12
+
13
+ # Default fallback env
14
+ DEFAULT_ENV = 'development'
15
+
16
+ private_constant :LOOKUP_ENV_KEYS, :DEFAULT_ENV
17
+
18
+ # @return [Karafka::Env] env object
19
+ # @note Will load appropriate environment automatically
20
+ def initialize
21
+ super('')
22
+
23
+ LOOKUP_ENV_KEYS
24
+ .map { |key| ENV[key] }
25
+ .compact
26
+ .first
27
+ .then { |env| env || DEFAULT_ENV }
28
+ .then { |env| replace(env) }
29
+ end
30
+
31
+ # @param method_name [String] method name
32
+ # @param include_private [Boolean] should we include private methods as well
33
+ # @return [Boolean] true if we respond to a given missing method, otherwise false
34
+ def respond_to_missing?(method_name, include_private = false)
35
+ (method_name[-1] == '?') || super
36
+ end
37
+
38
+ # Reacts to missing methods, from which some might be the env checks.
39
+ # If the method ends with '?' we assume, that it is an env check
40
+ # @param method_name [String] method name for missing or env name with question mark
41
+ # @param arguments [Array] any arguments that we pass to the method
42
+ def method_missing(method_name, *arguments)
43
+ method_name[-1] == '?' ? self == method_name[0..-2] : super
44
+ end
45
+ end
46
+ end
@@ -6,12 +6,6 @@ module Karafka
6
6
  # Base class for all the Karafka internal errors
7
7
  BaseError = Class.new(StandardError)
8
8
 
9
- # Should be raised when we have that that we cannot serialize
10
- SerializationError = Class.new(BaseError)
11
-
12
- # Should be raised when we tried to deserialize incoming data but we failed
13
- DeserializationError = Class.new(BaseError)
14
-
15
9
  # Raised when router receives topic name which does not correspond with any routes
16
10
  # This can only happen in a case when:
17
11
  # - you've received a message and we cannot match it with a consumer
@@ -24,28 +18,34 @@ module Karafka
24
18
  # @see https://github.com/karafka/karafka/issues/135
25
19
  NonMatchingRouteError = Class.new(BaseError)
26
20
 
27
- # Raised when we don't use or use responder not in the way it expected to based on the
28
- # topics usage definitions
29
- InvalidResponderUsageError = Class.new(BaseError)
30
-
31
- # Raised when options that we provide to the responder to respond aren't what the contract
32
- # requires
33
- InvalidResponderMessageOptionsError = Class.new(BaseError)
34
-
35
21
  # Raised when configuration doesn't match with validation contract
36
22
  InvalidConfigurationError = Class.new(BaseError)
37
23
 
38
24
  # Raised when we try to use Karafka CLI commands (except install) without a boot file
39
25
  MissingBootFileError = Class.new(BaseError)
40
26
 
41
- # Raised when we want to read a persisted thread messages consumer but it is unavailable
42
- # This should never happen and if it does, please contact us
43
- MissingClientError = Class.new(BaseError)
44
-
45
- # Raised when want to hook up to an event that is not registered and supported
46
- UnregisteredMonitorEventError = Class.new(BaseError)
47
-
48
27
  # Raised when we've waited enough for shutting down a non-responsive process
49
28
  ForcefulShutdownError = Class.new(BaseError)
29
+
30
+ # Raised when the jobs queue receives a job that should not be received as it would cause
31
+ # the processing to go out of sync. We should never process in parallel data from the same
32
+ # topic partition (unless virtual partitions apply)
33
+ JobsQueueSynchronizationError = Class.new(BaseError)
34
+
35
+ # Raised when given topic is not found while expected
36
+ TopicNotFoundError = Class.new(BaseError)
37
+
38
+ # This should never happen. Please open an issue if it does.
39
+ UnsupportedCaseError = Class.new(BaseError)
40
+
41
+ # Raised when the license token is not valid
42
+ InvalidLicenseTokenError = Class.new(BaseError)
43
+
44
+ # Used to instrument this error into the error notifications
45
+ # We do not raise it so we won't crash deployed systems
46
+ ExpiredLicenseTokenError = Class.new(BaseError)
47
+
48
+ # This should never happen. Please open an issue if it does.
49
+ InvalidCoordinatorState = Class.new(BaseError)
50
50
  end
51
51
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Helpers
5
+ # Allows a given class to run async in a separate thread. Provides also few methods we may
6
+ # want to use to control the underlying thread
7
+ #
8
+ # @note Thread running code needs to manage it's own exceptions. If they leak out, they will
9
+ # abort thread on exception.
10
+ module Async
11
+ class << self
12
+ # Adds forwardable to redirect thread-based control methods to the underlying thread that
13
+ # runs the async operations
14
+ #
15
+ # @param base [Class] class we're including this module in
16
+ def included(base)
17
+ base.extend ::Forwardable
18
+
19
+ base.def_delegators :@thread, :join, :terminate, :alive?
20
+ end
21
+ end
22
+
23
+ # Runs the `#call` method in a new thread
24
+ def async_call
25
+ @thread = Thread.new do
26
+ Thread.current.abort_on_exception = true
27
+
28
+ call
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Helpers
5
+ # Simple wrapper for adding colors to strings
6
+ module Colorize
7
+ # @param string [String] string we want to have in green
8
+ # @return [String] green string
9
+ def green(string)
10
+ "\033[0;32m#{string}\033[0m"
11
+ end
12
+
13
+ # @param string [String] string we want to have in red
14
+ # @return [String] red string
15
+ def red(string)
16
+ "\033[0;31m#{string}\033[0m"
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Karafka
4
- # Module containing classes and methods that provide some additional functionalities
4
+ # Module containing classes and methods that provide some additional helper functionalities.
5
5
  module Helpers
6
6
  # @note Taken from http://stackoverflow.com/questions/6407141
7
- # Multidelegator is used to delegate calls to multiple targets
7
+ # Multidelegator is used to delegate calls to multiple targets.
8
8
  class MultiDelegator
9
9
  # @param targets to which we want to delegate methods
10
10
  def initialize(*targets)
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Instrumentation
5
+ # Callbacks used to transport things from rdkafka
6
+ module Callbacks
7
+ # Callback that kicks in when consumer error occurs and is published in a background thread
8
+ class Error
9
+ # @param subscription_group_id [String] id of the current subscription group instance
10
+ # @param consumer_group_id [String] id of the current consumer group
11
+ # @param client_name [String] rdkafka client name
12
+ # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
13
+ def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
14
+ @subscription_group_id = subscription_group_id
15
+ @consumer_group_id = consumer_group_id
16
+ @client_name = client_name
17
+ @monitor = monitor
18
+ end
19
+
20
+ # Runs the instrumentation monitor with error
21
+ # @param client_name [String] rdkafka client name
22
+ # @param error [Rdkafka::Error] error that occurred
23
+ # @note It will only instrument on errors of the client of our consumer
24
+ def call(client_name, error)
25
+ # Emit only errors related to our client
26
+ # Same as with statistics (mor explanation there)
27
+ return unless @client_name == client_name
28
+
29
+ @monitor.instrument(
30
+ 'error.occurred',
31
+ subscription_group_id: @subscription_group_id,
32
+ consumer_group_id: @consumer_group_id,
33
+ type: 'librdkafka.error',
34
+ error: error
35
+ )
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end