rimless 1.1.1 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/documentation.yml +39 -0
  3. data/.github/workflows/test.yml +63 -0
  4. data/.rspec +2 -2
  5. data/.rubocop.yml +16 -2
  6. data/.simplecov +12 -0
  7. data/Appraisals +2 -22
  8. data/CHANGELOG.md +12 -0
  9. data/Dockerfile +2 -3
  10. data/Envfile +0 -3
  11. data/Guardfile +44 -0
  12. data/LICENSE +1 -1
  13. data/Makefile +25 -15
  14. data/README.md +11 -4
  15. data/Rakefile +13 -65
  16. data/doc/kafka-playground/.gitignore +2 -0
  17. data/doc/kafka-playground/Dockerfile +41 -0
  18. data/doc/kafka-playground/Gemfile +8 -0
  19. data/doc/kafka-playground/Gemfile.lock +155 -0
  20. data/doc/kafka-playground/Makefile +209 -0
  21. data/doc/kafka-playground/README.md +185 -0
  22. data/doc/kafka-playground/bin/consume-topic +7 -0
  23. data/doc/kafka-playground/bin/create-topic +42 -0
  24. data/doc/kafka-playground/bin/delete-topic +22 -0
  25. data/doc/kafka-playground/bin/list-topics +3 -0
  26. data/doc/kafka-playground/bin/produce-event +64 -0
  27. data/doc/kafka-playground/config/avro_schemas/.gitignore +1 -0
  28. data/doc/kafka-playground/config/avro_schemas/playground_app/item_v1.avsc.erb +36 -0
  29. data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1.avsc.erb +59 -0
  30. data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1_event.avsc.erb +18 -0
  31. data/doc/kafka-playground/config/docker/shell/.bash_profile +3 -0
  32. data/doc/kafka-playground/config/docker/shell/.bashrc +231 -0
  33. data/doc/kafka-playground/config/docker/shell/.config/kcat.conf +3 -0
  34. data/doc/kafka-playground/config/docker/shell/.gemrc +2 -0
  35. data/doc/kafka-playground/config/docker/shell/.inputrc +17 -0
  36. data/doc/kafka-playground/config/environment.rb +69 -0
  37. data/doc/kafka-playground/doc/assets/project.svg +68 -0
  38. data/doc/kafka-playground/docker-compose.yml +83 -0
  39. data/doc/kafka-playground/examples/rimless-produce +48 -0
  40. data/gemfiles/rails_5.2.gemfile +2 -2
  41. data/lib/rimless/configuration_handling.rb +11 -1
  42. data/lib/rimless/consumer.rb +4 -2
  43. data/lib/rimless/dependencies.rb +3 -0
  44. data/lib/rimless/kafka_helpers.rb +2 -0
  45. data/lib/rimless/karafka/avro_deserializer.rb +3 -3
  46. data/lib/rimless/rspec/helpers.rb +11 -0
  47. data/lib/rimless/rspec/matchers.rb +24 -9
  48. data/lib/rimless/rspec.rb +1 -1
  49. data/lib/rimless/tasks/consumer.rake +3 -0
  50. data/lib/rimless/tasks/generator.rake +3 -0
  51. data/lib/rimless/tasks/stats.rake +5 -2
  52. data/lib/rimless/version.rb +18 -1
  53. data/lib/rimless.rb +0 -1
  54. data/rimless.gemspec +43 -29
  55. metadata +121 -71
  56. data/.travis.yml +0 -35
  57. data/gemfiles/rails_4.2.gemfile +0 -8
  58. data/gemfiles/rails_5.0.gemfile +0 -8
  59. data/gemfiles/rails_5.1.gemfile +0 -8
  60. data/gemfiles/rails_6.0.gemfile +0 -8
@@ -4,6 +4,8 @@ module Rimless
4
4
  # The top-level configuration handling.
5
5
  #
6
6
  # rubocop:disable Style/ClassVars because we split module code
7
+ # rubocop:disable Metrics/BlockLength because this is how
8
+ # an +ActiveSupport::Concern+ looks like
7
9
  module ConfigurationHandling
8
10
  extend ActiveSupport::Concern
9
11
 
@@ -61,8 +63,15 @@ module Rimless
61
63
  # Check if a application is defined
62
64
  return if Rails.application.nil?
63
65
 
66
+ # We need a little compatibility here, as in Rails 6.1+ there is no
67
+ # more +parent_name+, instead they renamed it to +module_parent_name+
68
+ app_class = Rails.application.class
69
+ parent_name = app_class.module_parent_name \
70
+ if app_class.respond_to?(:module_parent_name)
71
+ parent_name ||= app_class.parent_name
72
+
64
73
  # Pass back the URI compatible application name
65
- Rails.application.class.parent_name.underscore.dasherize
74
+ parent_name.underscore.dasherize
66
75
  end
67
76
 
68
77
  # Retrieve the current configured logger instance.
@@ -72,4 +81,5 @@ module Rimless
72
81
  end
73
82
  end
74
83
  # rubocop:enable Style/ClassVars
84
+ # rubocop:enable Metrics/BlockLength
75
85
  end
@@ -47,7 +47,7 @@ module Rimless
47
47
  return unless rails_env.exist?
48
48
 
49
49
  ENV['RAILS_ENV'] ||= 'development'
50
- ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
50
+ ENV['KARAFKA_ENV'] = ENV.fetch('RAILS_ENV', nil)
51
51
  require rails_env
52
52
  Rails.application.eager_load!
53
53
  end
@@ -67,6 +67,7 @@ module Rimless
67
67
  # Configure the pure basics on the Karafka application.
68
68
  #
69
69
  # rubocop:disable Metrics/MethodLength because of the various settings
70
+ # rubocop:disable Metrics/AbcSize dito
70
71
  def initialize_karafka!
71
72
  setup do |config|
72
73
  mapper = Rimless::Karafka::PassthroughMapper.new
@@ -82,13 +83,14 @@ module Rimless
82
83
  end
83
84
  end
84
85
  # rubocop:enable Metrics/MethodLength
86
+ # rubocop:enable Metrics/AbcSize
85
87
 
86
88
  # When we run in development mode, we want to write the logs
87
89
  # to file and to stdout.
88
90
  def initialize_logger!
89
91
  return unless Rimless.env.development? && server?
90
92
 
91
- STDOUT.sync = true
93
+ $stdout.sync = true
92
94
  Rimless.logger.extend(ActiveSupport::Logger.broadcast(
93
95
  ActiveSupport::Logger.new($stdout)
94
96
  ))
@@ -17,6 +17,8 @@ module Rimless
17
17
  end
18
18
 
19
19
  # Set sensible defaults for the +WaterDrop+ gem.
20
+ #
21
+ # rubocop:disable Metrics/AbcSize because of the configuration mapping
20
22
  def configure_waterdrop
21
23
  # Skip WaterDrop configuration when no brokers/client id is available,
22
24
  # because it will raise. Its fine to have none available for situations
@@ -43,6 +45,7 @@ module Rimless
43
45
  config.kafka.required_acks = -1
44
46
  end
45
47
  end
48
+ # rubocop:enable Metrics/AbcSize
46
49
 
47
50
  # Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
48
51
  # Avro schema templates (ERB), so the gem can handle them properly.
@@ -21,6 +21,7 @@ module Rimless
21
21
  # Rimless.topic(name: 'test', app: :fancy_app)
22
22
  #
23
23
  # rubocop:disable Metrics/AbcSize because of the usage flexibility
24
+ # rubocop:disable Metrics/CyclomaticComplexity dito
24
25
  def topic(*args)
25
26
  opts = args.last
26
27
  name = args.first if [String, Symbol].member?(args.first.class)
@@ -38,6 +39,7 @@ module Rimless
38
39
  "#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
39
40
  end
40
41
  # rubocop:enable Metrics/AbcSize
42
+ # rubocop:enable Metrics/CyclomaticComplexity
41
43
 
42
44
  # Send a single message to Apache Kafka. The data is encoded according to
43
45
  # the given Apache Avro schema. The destination Kafka topic may be a
@@ -19,9 +19,9 @@ module Rimless
19
19
  # occurence should be rare.
20
20
  Rimless
21
21
  .decode(params.raw_payload)
22
- .yield_self { |data| Sparsify(data, sparse_array: true) }
23
- .yield_self { |data| data.transform_keys { |key| key.delete('\\') } }
24
- .yield_self { |data| Unsparsify(data, sparse_array: true) }
22
+ .then { |data| Sparsify(data, sparse_array: true) }
23
+ .then { |data| data.transform_keys { |key| key.delete('\\') } }
24
+ .then { |data| Unsparsify(data, sparse_array: true) }
25
25
  .deep_symbolize_keys
26
26
  end
27
27
  end
@@ -23,6 +23,8 @@ module Rimless
23
23
  # @return [OpenStruct] the fake deserialized Kafka message
24
24
  #
25
25
  # rubocop:disable Metrics/MethodLength because of the various properties
26
+ # rubocop:disable Style/OpenStructUse because existing specs may rely
27
+ # on this data type
26
28
  def kafka_message(topic: nil, headers: {}, **payload)
27
29
  OpenStruct.new(
28
30
  topic: Rimless.topic(topic),
@@ -38,6 +40,15 @@ module Rimless
38
40
  )
39
41
  end
40
42
  # rubocop:enable Metrics/MethodLength
43
+ # rubocop:enable Style/OpenStructUse
44
+
45
+ # Capture all Apache Kafka messages of the given block.
46
+ #
47
+ # @yield the given block to capture the messages
48
+ # @return [Array<Hash{Symbol => Mixed}>] the captured messages
49
+ def capture_kafka_messages(&block)
50
+ Rimless::RSpec::Matchers::HaveSentKafkaMessage.new(nil).capture(&block)
51
+ end
41
52
  end
42
53
  end
43
54
  end
@@ -16,6 +16,7 @@ module Rimless
16
16
  # @param schema [String, Symbol, nil] the expected message schema
17
17
  # @return [HaveSentKafkaMessage] the expectation instance
18
18
  def initialize(schema)
19
+ super
19
20
  @schema = schema
20
21
  @args = {}
21
22
  @data = {}
@@ -23,6 +24,15 @@ module Rimless
23
24
  set_expected_number(:exactly, 1)
24
25
  end
25
26
 
27
+ # Capture all Apache Kafka messages of the given block.
28
+ #
29
+ # @yield the given block to capture the messages
30
+ # @return [Array<Hash{Symbol => Mixed}>] the captured messages
31
+ def capture(&block)
32
+ matches?(block)
33
+ @messages
34
+ end
35
+
26
36
  # Collect the expectation arguments for the Kafka message passing. (eg.
27
37
  # topic)
28
38
  #
@@ -173,8 +183,9 @@ module Rimless
173
183
  return true unless @schema
174
184
 
175
185
  begin
176
- Rimless.avro.decode(message[:data], schema_name: @schema.to_s)
177
- return true
186
+ Rimless.avro.decode(message[:encoded_data],
187
+ schema_name: @schema.to_s)
188
+ true
178
189
  rescue Avro::IO::SchemaMatchException
179
190
  false
180
191
  end
@@ -199,24 +210,28 @@ module Rimless
199
210
  def data_match?(message)
200
211
  return true unless @data.any?
201
212
 
202
- actual_data = Rimless.avro.decode(message[:data])
203
- expected_data = @data.deep_stringify_keys
204
-
205
- actual_data.merge(expected_data) == actual_data
213
+ message[:data].merge(@data.deep_stringify_keys) == message[:data]
206
214
  end
207
215
 
208
216
  # Setup the +WaterDrop+ spies and record each sent message.
217
+ # because of the message decoding
218
+ # rubocop:disable Metrics/MethodLength dito
209
219
  def listen_to_messages
220
+ decode = proc do |encoded|
221
+ { encoded_data: encoded, data: Rimless.avro.decode(encoded) }
222
+ end
223
+
210
224
  allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
211
- @messages << { data: data, args: args, type: :sync }
225
+ @messages << { args: args, type: :sync }.merge(decode[data])
212
226
  nil
213
227
  end
214
228
 
215
229
  allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
216
- @messages << { data: data, args: args, type: :async }
230
+ @messages << { args: args, type: :async }.merge(decode[data])
217
231
  nil
218
232
  end
219
233
  end
234
+ # rubocop:enable Metrics/MethodLength
220
235
 
221
236
  # Serve the RSpec API and return the positive failure message.
222
237
  #
@@ -264,7 +279,7 @@ module Rimless
264
279
  result = ['message']
265
280
 
266
281
  result << " with #{message[:args]}" if message[:args].any?
267
- result << " with data: #{Rimless.avro.decode(message[:data])}"
282
+ result << " with data: #{message[:data]}"
268
283
 
269
284
  result.join
270
285
  end
data/lib/rimless/rspec.rb CHANGED
@@ -33,7 +33,7 @@ RSPEC_CONFIGURER.configure do |config|
33
33
  # schema repository it cannot conflict while refreshing it.
34
34
  unless ENV['TEST_ENV_NUMBER'].nil?
35
35
  Rimless.configure do |conf|
36
- num = ENV['TEST_ENV_NUMBER']
36
+ num = ENV.fetch('TEST_ENV_NUMBER', nil)
37
37
  num = '1' if num.empty?
38
38
 
39
39
  conf.compiled_avro_schema_path = \
@@ -1,10 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  namespace :rimless do
4
+ # rubocop:disable Rails/RakeEnvironment because this is just an command
5
+ # proxy, no need for an application bootstrap
4
6
  desc 'Start the Apache Kafka consumer'
5
7
  task :consumer do
6
8
  system 'bundle exec karafka server'
7
9
  end
10
+ # rubocop:enable Rails/RakeEnvironment
8
11
 
9
12
  desc 'Print all the consumer routes'
10
13
  task routes: :environment do
@@ -18,6 +18,8 @@ namespace :rimless do
18
18
  FileUtils.copy(src, dest)
19
19
  end
20
20
 
21
+ # rubocop:disable Rails/RakeEnvironment because this is just an
22
+ # helper command, no need for an application bootstrap
21
23
  desc 'Install the Rimless consumer components'
22
24
  task :install do
23
25
  install_template('karafka.rb')
@@ -33,4 +35,5 @@ namespace :rimless do
33
35
  # your project root. And list all routes with +rake rimless:routes+.
34
36
  OUTPUT
35
37
  end
38
+ # rubocop:enable Rails/RakeEnvironment
36
39
  end
@@ -3,6 +3,8 @@
3
3
  if defined?(Rails) && Rails.env.development?
4
4
  require 'rspec/core/rake_task'
5
5
 
6
+ # rubocop:disable Rails/RakeEnvironment because this is just an
7
+ # helper command, no need for an application bootstrap
6
8
  task :stats do
7
9
  require 'rails/code_statistics'
8
10
 
@@ -11,8 +13,9 @@ if defined?(Rails) && Rails.env.development?
11
13
  ].each do |method, type, dir|
12
14
  next unless File.directory? dir
13
15
 
14
- ::STATS_DIRECTORIES.send(method, [type, dir])
15
- ::CodeStatistics::TEST_TYPES << type if type.include? 'specs'
16
+ STATS_DIRECTORIES.send(method, [type, dir])
17
+ CodeStatistics::TEST_TYPES << type if type.include? 'specs'
16
18
  end
17
19
  end
20
+ # rubocop:enable Rails/RakeEnvironment
18
21
  end
@@ -1,6 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # The gem version details.
3
4
  module Rimless
4
5
  # The version of the +rimless+ gem
5
- VERSION = '1.1.1'
6
+ VERSION = '1.3.0'
7
+
8
+ class << self
9
+ # Returns the version of gem as a string.
10
+ #
11
+ # @return [String] the gem version as string
12
+ def version
13
+ VERSION
14
+ end
15
+
16
+ # Returns the version of the gem as a +Gem::Version+.
17
+ #
18
+ # @return [Gem::Version] the gem version as object
19
+ def gem_version
20
+ Gem::Version.new VERSION
21
+ end
22
+ end
6
23
  end
data/lib/rimless.rb CHANGED
@@ -15,7 +15,6 @@ require 'karafka'
15
15
  require 'karafka-sidekiq-backend'
16
16
  require 'sparsify'
17
17
  require 'erb'
18
- require 'pp'
19
18
 
20
19
  # The top level namespace for the rimless gem.
21
20
  module Rimless
data/rimless.gemspec CHANGED
@@ -5,47 +5,61 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require 'rimless/version'
6
6
 
7
7
  Gem::Specification.new do |spec|
8
- spec.name = 'rimless'
9
- spec.version = Rimless::VERSION
10
- spec.authors = ['Hermann Mayer']
11
- spec.email = ['hermann.mayer92@gmail.com']
8
+ spec.name = 'rimless'
9
+ spec.version = Rimless::VERSION
10
+ spec.authors = ['Hermann Mayer']
11
+ spec.email = ['hermann.mayer92@gmail.com']
12
12
 
13
- spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
14
- 'Schema Registry helpers.'
15
- spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
16
- 'Schema Registry helpers.'
17
- spec.homepage = 'https://github.com/hausgold/rimless'
13
+ spec.license = 'MIT'
14
+ spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
15
+ 'Schema Registry helpers.'
16
+ spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
17
+ 'Schema Registry helpers.'
18
+
19
+ base_uri = "https://github.com/hausgold/#{spec.name}"
20
+ spec.metadata = {
21
+ 'homepage_uri' => base_uri,
22
+ 'source_code_uri' => base_uri,
23
+ 'changelog_uri' => "#{base_uri}/blob/master/CHANGELOG.md",
24
+ 'bug_tracker_uri' => "#{base_uri}/issues",
25
+ 'documentation_uri' => "https://www.rubydoc.info/gems/#{spec.name}"
26
+ }
18
27
 
19
28
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
20
29
  f.match(%r{^(test|spec|features)/})
21
30
  end
22
- spec.bindir = 'exe'
23
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
31
+
32
+ spec.bindir = 'exe'
33
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
24
34
  spec.require_paths = ['lib']
25
35
 
26
- spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
36
+ spec.required_ruby_version = '>= 2.5'
37
+
38
+ spec.add_runtime_dependency 'activesupport', '>= 5.2'
27
39
  spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
28
40
  spec.add_runtime_dependency 'karafka', '~> 1.4'
29
41
  spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.4'
30
42
  spec.add_runtime_dependency 'karafka-testing', '~> 1.4'
31
- spec.add_runtime_dependency 'sinatra'
43
+ spec.add_runtime_dependency 'sinatra', '~> 2.2'
32
44
  spec.add_runtime_dependency 'sparsify', '~> 1.1'
33
- spec.add_runtime_dependency 'waterdrop', '~> 1.2'
34
- spec.add_runtime_dependency 'webmock'
45
+ spec.add_runtime_dependency 'waterdrop', '~> 1.4'
46
+ spec.add_runtime_dependency 'webmock', '~> 3.18'
35
47
 
36
- spec.add_development_dependency 'appraisal'
37
- spec.add_development_dependency 'bundler', '>= 1.16', '< 3'
38
- spec.add_development_dependency 'factory_bot', '~> 4.11'
39
- spec.add_development_dependency 'railties', '>= 4.2.0'
48
+ spec.add_development_dependency 'appraisal', '~> 2.4'
49
+ spec.add_development_dependency 'bundler', '~> 2.3'
50
+ spec.add_development_dependency 'countless', '~> 1.1'
51
+ spec.add_development_dependency 'factory_bot', '~> 6.2'
52
+ spec.add_development_dependency 'guard-rspec', '~> 4.7'
53
+ spec.add_development_dependency 'railties', '>= 5.2'
40
54
  spec.add_development_dependency 'rake', '~> 13.0'
41
- spec.add_development_dependency 'rdoc', '~> 6.1'
42
- spec.add_development_dependency 'redcarpet', '~> 3.4'
43
- spec.add_development_dependency 'rspec', '~> 3.0'
44
- spec.add_development_dependency 'rubocop', '~> 0.63.1'
45
- spec.add_development_dependency 'rubocop-rspec', '~> 1.31'
46
- spec.add_development_dependency 'simplecov', '~> 0.15'
47
- spec.add_development_dependency 'timecop', '~> 0.9.1'
48
- spec.add_development_dependency 'vcr', '~> 3.0'
49
- spec.add_development_dependency 'yard', '~> 0.9.18'
50
- spec.add_development_dependency 'yard-activesupport-concern', '~> 0.0.1'
55
+ spec.add_development_dependency 'redcarpet', '~> 3.5'
56
+ spec.add_development_dependency 'rspec', '~> 3.12'
57
+ spec.add_development_dependency 'rubocop', '~> 1.28'
58
+ spec.add_development_dependency 'rubocop-rails', '~> 2.14'
59
+ spec.add_development_dependency 'rubocop-rspec', '~> 2.10'
60
+ spec.add_development_dependency 'simplecov', '>= 0.22'
61
+ spec.add_development_dependency 'timecop', '>= 0.9.6'
62
+ spec.add_development_dependency 'vcr', '~> 6.0'
63
+ spec.add_development_dependency 'yard', '>= 0.9.28'
64
+ spec.add_development_dependency 'yard-activesupport-concern', '>= 0.0.1'
51
65
  end