rimless 1.1.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/documentation.yml +39 -0
  3. data/.github/workflows/test.yml +63 -0
  4. data/.rspec +2 -2
  5. data/.rubocop.yml +16 -2
  6. data/.simplecov +12 -0
  7. data/Appraisals +2 -22
  8. data/CHANGELOG.md +12 -0
  9. data/Dockerfile +2 -3
  10. data/Envfile +0 -3
  11. data/Guardfile +44 -0
  12. data/LICENSE +1 -1
  13. data/Makefile +25 -15
  14. data/README.md +11 -4
  15. data/Rakefile +13 -65
  16. data/doc/kafka-playground/.gitignore +2 -0
  17. data/doc/kafka-playground/Dockerfile +41 -0
  18. data/doc/kafka-playground/Gemfile +8 -0
  19. data/doc/kafka-playground/Gemfile.lock +155 -0
  20. data/doc/kafka-playground/Makefile +209 -0
  21. data/doc/kafka-playground/README.md +185 -0
  22. data/doc/kafka-playground/bin/consume-topic +7 -0
  23. data/doc/kafka-playground/bin/create-topic +42 -0
  24. data/doc/kafka-playground/bin/delete-topic +22 -0
  25. data/doc/kafka-playground/bin/list-topics +3 -0
  26. data/doc/kafka-playground/bin/produce-event +64 -0
  27. data/doc/kafka-playground/config/avro_schemas/.gitignore +1 -0
  28. data/doc/kafka-playground/config/avro_schemas/playground_app/item_v1.avsc.erb +36 -0
  29. data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1.avsc.erb +59 -0
  30. data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1_event.avsc.erb +18 -0
  31. data/doc/kafka-playground/config/docker/shell/.bash_profile +3 -0
  32. data/doc/kafka-playground/config/docker/shell/.bashrc +231 -0
  33. data/doc/kafka-playground/config/docker/shell/.config/kcat.conf +3 -0
  34. data/doc/kafka-playground/config/docker/shell/.gemrc +2 -0
  35. data/doc/kafka-playground/config/docker/shell/.inputrc +17 -0
  36. data/doc/kafka-playground/config/environment.rb +69 -0
  37. data/doc/kafka-playground/doc/assets/project.svg +68 -0
  38. data/doc/kafka-playground/docker-compose.yml +83 -0
  39. data/doc/kafka-playground/examples/rimless-produce +48 -0
  40. data/gemfiles/rails_5.2.gemfile +2 -2
  41. data/lib/rimless/configuration_handling.rb +11 -1
  42. data/lib/rimless/consumer.rb +4 -2
  43. data/lib/rimless/dependencies.rb +3 -0
  44. data/lib/rimless/kafka_helpers.rb +2 -0
  45. data/lib/rimless/karafka/avro_deserializer.rb +3 -3
  46. data/lib/rimless/rspec/helpers.rb +11 -0
  47. data/lib/rimless/rspec/matchers.rb +24 -9
  48. data/lib/rimless/rspec.rb +1 -1
  49. data/lib/rimless/tasks/consumer.rake +3 -0
  50. data/lib/rimless/tasks/generator.rake +3 -0
  51. data/lib/rimless/tasks/stats.rake +5 -2
  52. data/lib/rimless/version.rb +18 -1
  53. data/lib/rimless.rb +0 -1
  54. data/rimless.gemspec +43 -29
  55. metadata +121 -71
  56. data/.travis.yml +0 -35
  57. data/gemfiles/rails_4.2.gemfile +0 -8
  58. data/gemfiles/rails_5.0.gemfile +0 -8
  59. data/gemfiles/rails_5.1.gemfile +0 -8
  60. data/gemfiles/rails_6.0.gemfile +0 -8
@@ -4,6 +4,8 @@ module Rimless
4
4
  # The top-level configuration handling.
5
5
  #
6
6
  # rubocop:disable Style/ClassVars because we split module code
7
+ # rubocop:disable Metrics/BlockLength because this is how
8
+ # an +ActiveSupport::Concern+ looks like
7
9
  module ConfigurationHandling
8
10
  extend ActiveSupport::Concern
9
11
 
@@ -61,8 +63,15 @@ module Rimless
61
63
  # Check if a application is defined
62
64
  return if Rails.application.nil?
63
65
 
66
+ # We need a little compatibility here, as in Rails 6.1+ there is no
67
+ # more +parent_name+, instead they renamed it to +module_parent_name+
68
+ app_class = Rails.application.class
69
+ parent_name = app_class.module_parent_name \
70
+ if app_class.respond_to?(:module_parent_name)
71
+ parent_name ||= app_class.parent_name
72
+
64
73
  # Pass back the URI compatible application name
65
- Rails.application.class.parent_name.underscore.dasherize
74
+ parent_name.underscore.dasherize
66
75
  end
67
76
 
68
77
  # Retrieve the current configured logger instance.
@@ -72,4 +81,5 @@ module Rimless
72
81
  end
73
82
  end
74
83
  # rubocop:enable Style/ClassVars
84
+ # rubocop:enable Metrics/BlockLength
75
85
  end
@@ -47,7 +47,7 @@ module Rimless
47
47
  return unless rails_env.exist?
48
48
 
49
49
  ENV['RAILS_ENV'] ||= 'development'
50
- ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
50
+ ENV['KARAFKA_ENV'] = ENV.fetch('RAILS_ENV', nil)
51
51
  require rails_env
52
52
  Rails.application.eager_load!
53
53
  end
@@ -67,6 +67,7 @@ module Rimless
67
67
  # Configure the pure basics on the Karafka application.
68
68
  #
69
69
  # rubocop:disable Metrics/MethodLength because of the various settings
70
+ # rubocop:disable Metrics/AbcSize dito
70
71
  def initialize_karafka!
71
72
  setup do |config|
72
73
  mapper = Rimless::Karafka::PassthroughMapper.new
@@ -82,13 +83,14 @@ module Rimless
82
83
  end
83
84
  end
84
85
  # rubocop:enable Metrics/MethodLength
86
+ # rubocop:enable Metrics/AbcSize
85
87
 
86
88
  # When we run in development mode, we want to write the logs
87
89
  # to file and to stdout.
88
90
  def initialize_logger!
89
91
  return unless Rimless.env.development? && server?
90
92
 
91
- STDOUT.sync = true
93
+ $stdout.sync = true
92
94
  Rimless.logger.extend(ActiveSupport::Logger.broadcast(
93
95
  ActiveSupport::Logger.new($stdout)
94
96
  ))
@@ -17,6 +17,8 @@ module Rimless
17
17
  end
18
18
 
19
19
  # Set sensible defaults for the +WaterDrop+ gem.
20
+ #
21
+ # rubocop:disable Metrics/AbcSize because of the configuration mapping
20
22
  def configure_waterdrop
21
23
  # Skip WaterDrop configuration when no brokers/client id is available,
22
24
  # because it will raise. Its fine to have none available for situations
@@ -43,6 +45,7 @@ module Rimless
43
45
  config.kafka.required_acks = -1
44
46
  end
45
47
  end
48
+ # rubocop:enable Metrics/AbcSize
46
49
 
47
50
  # Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
48
51
  # Avro schema templates (ERB), so the gem can handle them properly.
@@ -21,6 +21,7 @@ module Rimless
21
21
  # Rimless.topic(name: 'test', app: :fancy_app)
22
22
  #
23
23
  # rubocop:disable Metrics/AbcSize because of the usage flexibility
24
+ # rubocop:disable Metrics/CyclomaticComplexity dito
24
25
  def topic(*args)
25
26
  opts = args.last
26
27
  name = args.first if [String, Symbol].member?(args.first.class)
@@ -38,6 +39,7 @@ module Rimless
38
39
  "#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
39
40
  end
40
41
  # rubocop:enable Metrics/AbcSize
42
+ # rubocop:enable Metrics/CyclomaticComplexity
41
43
 
42
44
  # Send a single message to Apache Kafka. The data is encoded according to
43
45
  # the given Apache Avro schema. The destination Kafka topic may be a
@@ -19,9 +19,9 @@ module Rimless
19
19
  # occurence should be rare.
20
20
  Rimless
21
21
  .decode(params.raw_payload)
22
- .yield_self { |data| Sparsify(data, sparse_array: true) }
23
- .yield_self { |data| data.transform_keys { |key| key.delete('\\') } }
24
- .yield_self { |data| Unsparsify(data, sparse_array: true) }
22
+ .then { |data| Sparsify(data, sparse_array: true) }
23
+ .then { |data| data.transform_keys { |key| key.delete('\\') } }
24
+ .then { |data| Unsparsify(data, sparse_array: true) }
25
25
  .deep_symbolize_keys
26
26
  end
27
27
  end
@@ -23,6 +23,8 @@ module Rimless
23
23
  # @return [OpenStruct] the fake deserialized Kafka message
24
24
  #
25
25
  # rubocop:disable Metrics/MethodLength because of the various properties
26
+ # rubocop:disable Style/OpenStructUse because existing specs may rely
27
+ # on this data type
26
28
  def kafka_message(topic: nil, headers: {}, **payload)
27
29
  OpenStruct.new(
28
30
  topic: Rimless.topic(topic),
@@ -38,6 +40,15 @@ module Rimless
38
40
  )
39
41
  end
40
42
  # rubocop:enable Metrics/MethodLength
43
+ # rubocop:enable Style/OpenStructUse
44
+
45
+ # Capture all Apache Kafka messages of the given block.
46
+ #
47
+ # @yield the given block to capture the messages
48
+ # @return [Array<Hash{Symbol => Mixed}>] the captured messages
49
+ def capture_kafka_messages(&block)
50
+ Rimless::RSpec::Matchers::HaveSentKafkaMessage.new(nil).capture(&block)
51
+ end
41
52
  end
42
53
  end
43
54
  end
@@ -16,6 +16,7 @@ module Rimless
16
16
  # @param schema [String, Symbol, nil] the expected message schema
17
17
  # @return [HaveSentKafkaMessage] the expectation instance
18
18
  def initialize(schema)
19
+ super
19
20
  @schema = schema
20
21
  @args = {}
21
22
  @data = {}
@@ -23,6 +24,15 @@ module Rimless
23
24
  set_expected_number(:exactly, 1)
24
25
  end
25
26
 
27
+ # Capture all Apache Kafka messages of the given block.
28
+ #
29
+ # @yield the given block to capture the messages
30
+ # @return [Array<Hash{Symbol => Mixed}>] the captured messages
31
+ def capture(&block)
32
+ matches?(block)
33
+ @messages
34
+ end
35
+
26
36
  # Collect the expectation arguments for the Kafka message passing. (eg.
27
37
  # topic)
28
38
  #
@@ -173,8 +183,9 @@ module Rimless
173
183
  return true unless @schema
174
184
 
175
185
  begin
176
- Rimless.avro.decode(message[:data], schema_name: @schema.to_s)
177
- return true
186
+ Rimless.avro.decode(message[:encoded_data],
187
+ schema_name: @schema.to_s)
188
+ true
178
189
  rescue Avro::IO::SchemaMatchException
179
190
  false
180
191
  end
@@ -199,24 +210,28 @@ module Rimless
199
210
  def data_match?(message)
200
211
  return true unless @data.any?
201
212
 
202
- actual_data = Rimless.avro.decode(message[:data])
203
- expected_data = @data.deep_stringify_keys
204
-
205
- actual_data.merge(expected_data) == actual_data
213
+ message[:data].merge(@data.deep_stringify_keys) == message[:data]
206
214
  end
207
215
 
208
216
  # Setup the +WaterDrop+ spies and record each sent message.
217
+ # because of the message decoding
218
+ # rubocop:disable Metrics/MethodLength dito
209
219
  def listen_to_messages
220
+ decode = proc do |encoded|
221
+ { encoded_data: encoded, data: Rimless.avro.decode(encoded) }
222
+ end
223
+
210
224
  allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
211
- @messages << { data: data, args: args, type: :sync }
225
+ @messages << { args: args, type: :sync }.merge(decode[data])
212
226
  nil
213
227
  end
214
228
 
215
229
  allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
216
- @messages << { data: data, args: args, type: :async }
230
+ @messages << { args: args, type: :async }.merge(decode[data])
217
231
  nil
218
232
  end
219
233
  end
234
+ # rubocop:enable Metrics/MethodLength
220
235
 
221
236
  # Serve the RSpec API and return the positive failure message.
222
237
  #
@@ -264,7 +279,7 @@ module Rimless
264
279
  result = ['message']
265
280
 
266
281
  result << " with #{message[:args]}" if message[:args].any?
267
- result << " with data: #{Rimless.avro.decode(message[:data])}"
282
+ result << " with data: #{message[:data]}"
268
283
 
269
284
  result.join
270
285
  end
data/lib/rimless/rspec.rb CHANGED
@@ -33,7 +33,7 @@ RSPEC_CONFIGURER.configure do |config|
33
33
  # schema repository it cannot conflict while refreshing it.
34
34
  unless ENV['TEST_ENV_NUMBER'].nil?
35
35
  Rimless.configure do |conf|
36
- num = ENV['TEST_ENV_NUMBER']
36
+ num = ENV.fetch('TEST_ENV_NUMBER', nil)
37
37
  num = '1' if num.empty?
38
38
 
39
39
  conf.compiled_avro_schema_path = \
@@ -1,10 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  namespace :rimless do
4
+ # rubocop:disable Rails/RakeEnvironment because this is just an command
5
+ # proxy, no need for an application bootstrap
4
6
  desc 'Start the Apache Kafka consumer'
5
7
  task :consumer do
6
8
  system 'bundle exec karafka server'
7
9
  end
10
+ # rubocop:enable Rails/RakeEnvironment
8
11
 
9
12
  desc 'Print all the consumer routes'
10
13
  task routes: :environment do
@@ -18,6 +18,8 @@ namespace :rimless do
18
18
  FileUtils.copy(src, dest)
19
19
  end
20
20
 
21
+ # rubocop:disable Rails/RakeEnvironment because this is just an
22
+ # helper command, no need for an application bootstrap
21
23
  desc 'Install the Rimless consumer components'
22
24
  task :install do
23
25
  install_template('karafka.rb')
@@ -33,4 +35,5 @@ namespace :rimless do
33
35
  # your project root. And list all routes with +rake rimless:routes+.
34
36
  OUTPUT
35
37
  end
38
+ # rubocop:enable Rails/RakeEnvironment
36
39
  end
@@ -3,6 +3,8 @@
3
3
  if defined?(Rails) && Rails.env.development?
4
4
  require 'rspec/core/rake_task'
5
5
 
6
+ # rubocop:disable Rails/RakeEnvironment because this is just an
7
+ # helper command, no need for an application bootstrap
6
8
  task :stats do
7
9
  require 'rails/code_statistics'
8
10
 
@@ -11,8 +13,9 @@ if defined?(Rails) && Rails.env.development?
11
13
  ].each do |method, type, dir|
12
14
  next unless File.directory? dir
13
15
 
14
- ::STATS_DIRECTORIES.send(method, [type, dir])
15
- ::CodeStatistics::TEST_TYPES << type if type.include? 'specs'
16
+ STATS_DIRECTORIES.send(method, [type, dir])
17
+ CodeStatistics::TEST_TYPES << type if type.include? 'specs'
16
18
  end
17
19
  end
20
+ # rubocop:enable Rails/RakeEnvironment
18
21
  end
@@ -1,6 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # The gem version details.
3
4
  module Rimless
4
5
  # The version of the +rimless+ gem
5
- VERSION = '1.1.1'
6
+ VERSION = '1.3.0'
7
+
8
+ class << self
9
+ # Returns the version of gem as a string.
10
+ #
11
+ # @return [String] the gem version as string
12
+ def version
13
+ VERSION
14
+ end
15
+
16
+ # Returns the version of the gem as a +Gem::Version+.
17
+ #
18
+ # @return [Gem::Version] the gem version as object
19
+ def gem_version
20
+ Gem::Version.new VERSION
21
+ end
22
+ end
6
23
  end
data/lib/rimless.rb CHANGED
@@ -15,7 +15,6 @@ require 'karafka'
15
15
  require 'karafka-sidekiq-backend'
16
16
  require 'sparsify'
17
17
  require 'erb'
18
- require 'pp'
19
18
 
20
19
  # The top level namespace for the rimless gem.
21
20
  module Rimless
data/rimless.gemspec CHANGED
@@ -5,47 +5,61 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require 'rimless/version'
6
6
 
7
7
  Gem::Specification.new do |spec|
8
- spec.name = 'rimless'
9
- spec.version = Rimless::VERSION
10
- spec.authors = ['Hermann Mayer']
11
- spec.email = ['hermann.mayer92@gmail.com']
8
+ spec.name = 'rimless'
9
+ spec.version = Rimless::VERSION
10
+ spec.authors = ['Hermann Mayer']
11
+ spec.email = ['hermann.mayer92@gmail.com']
12
12
 
13
- spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
14
- 'Schema Registry helpers.'
15
- spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
16
- 'Schema Registry helpers.'
17
- spec.homepage = 'https://github.com/hausgold/rimless'
13
+ spec.license = 'MIT'
14
+ spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
15
+ 'Schema Registry helpers.'
16
+ spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
17
+ 'Schema Registry helpers.'
18
+
19
+ base_uri = "https://github.com/hausgold/#{spec.name}"
20
+ spec.metadata = {
21
+ 'homepage_uri' => base_uri,
22
+ 'source_code_uri' => base_uri,
23
+ 'changelog_uri' => "#{base_uri}/blob/master/CHANGELOG.md",
24
+ 'bug_tracker_uri' => "#{base_uri}/issues",
25
+ 'documentation_uri' => "https://www.rubydoc.info/gems/#{spec.name}"
26
+ }
18
27
 
19
28
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
20
29
  f.match(%r{^(test|spec|features)/})
21
30
  end
22
- spec.bindir = 'exe'
23
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
31
+
32
+ spec.bindir = 'exe'
33
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
24
34
  spec.require_paths = ['lib']
25
35
 
26
- spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
36
+ spec.required_ruby_version = '>= 2.5'
37
+
38
+ spec.add_runtime_dependency 'activesupport', '>= 5.2'
27
39
  spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
28
40
  spec.add_runtime_dependency 'karafka', '~> 1.4'
29
41
  spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.4'
30
42
  spec.add_runtime_dependency 'karafka-testing', '~> 1.4'
31
- spec.add_runtime_dependency 'sinatra'
43
+ spec.add_runtime_dependency 'sinatra', '~> 2.2'
32
44
  spec.add_runtime_dependency 'sparsify', '~> 1.1'
33
- spec.add_runtime_dependency 'waterdrop', '~> 1.2'
34
- spec.add_runtime_dependency 'webmock'
45
+ spec.add_runtime_dependency 'waterdrop', '~> 1.4'
46
+ spec.add_runtime_dependency 'webmock', '~> 3.18'
35
47
 
36
- spec.add_development_dependency 'appraisal'
37
- spec.add_development_dependency 'bundler', '>= 1.16', '< 3'
38
- spec.add_development_dependency 'factory_bot', '~> 4.11'
39
- spec.add_development_dependency 'railties', '>= 4.2.0'
48
+ spec.add_development_dependency 'appraisal', '~> 2.4'
49
+ spec.add_development_dependency 'bundler', '~> 2.3'
50
+ spec.add_development_dependency 'countless', '~> 1.1'
51
+ spec.add_development_dependency 'factory_bot', '~> 6.2'
52
+ spec.add_development_dependency 'guard-rspec', '~> 4.7'
53
+ spec.add_development_dependency 'railties', '>= 5.2'
40
54
  spec.add_development_dependency 'rake', '~> 13.0'
41
- spec.add_development_dependency 'rdoc', '~> 6.1'
42
- spec.add_development_dependency 'redcarpet', '~> 3.4'
43
- spec.add_development_dependency 'rspec', '~> 3.0'
44
- spec.add_development_dependency 'rubocop', '~> 0.63.1'
45
- spec.add_development_dependency 'rubocop-rspec', '~> 1.31'
46
- spec.add_development_dependency 'simplecov', '~> 0.15'
47
- spec.add_development_dependency 'timecop', '~> 0.9.1'
48
- spec.add_development_dependency 'vcr', '~> 3.0'
49
- spec.add_development_dependency 'yard', '~> 0.9.18'
50
- spec.add_development_dependency 'yard-activesupport-concern', '~> 0.0.1'
55
+ spec.add_development_dependency 'redcarpet', '~> 3.5'
56
+ spec.add_development_dependency 'rspec', '~> 3.12'
57
+ spec.add_development_dependency 'rubocop', '~> 1.28'
58
+ spec.add_development_dependency 'rubocop-rails', '~> 2.14'
59
+ spec.add_development_dependency 'rubocop-rspec', '~> 2.10'
60
+ spec.add_development_dependency 'simplecov', '>= 0.22'
61
+ spec.add_development_dependency 'timecop', '>= 0.9.6'
62
+ spec.add_development_dependency 'vcr', '~> 6.0'
63
+ spec.add_development_dependency 'yard', '>= 0.9.28'
64
+ spec.add_development_dependency 'yard-activesupport-concern', '>= 0.0.1'
51
65
  end