rimless 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/documentation.yml +39 -0
- data/.github/workflows/test.yml +63 -0
- data/.rspec +2 -2
- data/.rubocop.yml +16 -2
- data/.simplecov +12 -0
- data/Appraisals +2 -22
- data/CHANGELOG.md +12 -0
- data/Dockerfile +2 -3
- data/Envfile +0 -3
- data/Guardfile +44 -0
- data/LICENSE +1 -1
- data/Makefile +25 -15
- data/README.md +11 -4
- data/Rakefile +13 -65
- data/doc/kafka-playground/.gitignore +2 -0
- data/doc/kafka-playground/Dockerfile +41 -0
- data/doc/kafka-playground/Gemfile +8 -0
- data/doc/kafka-playground/Gemfile.lock +155 -0
- data/doc/kafka-playground/Makefile +209 -0
- data/doc/kafka-playground/README.md +185 -0
- data/doc/kafka-playground/bin/consume-topic +7 -0
- data/doc/kafka-playground/bin/create-topic +42 -0
- data/doc/kafka-playground/bin/delete-topic +22 -0
- data/doc/kafka-playground/bin/list-topics +3 -0
- data/doc/kafka-playground/bin/produce-event +64 -0
- data/doc/kafka-playground/config/avro_schemas/.gitignore +1 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/item_v1.avsc.erb +36 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1.avsc.erb +59 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1_event.avsc.erb +18 -0
- data/doc/kafka-playground/config/docker/shell/.bash_profile +3 -0
- data/doc/kafka-playground/config/docker/shell/.bashrc +231 -0
- data/doc/kafka-playground/config/docker/shell/.config/kcat.conf +3 -0
- data/doc/kafka-playground/config/docker/shell/.gemrc +2 -0
- data/doc/kafka-playground/config/docker/shell/.inputrc +17 -0
- data/doc/kafka-playground/config/environment.rb +69 -0
- data/doc/kafka-playground/doc/assets/project.svg +68 -0
- data/doc/kafka-playground/docker-compose.yml +83 -0
- data/doc/kafka-playground/examples/rimless-produce +48 -0
- data/gemfiles/rails_5.2.gemfile +2 -2
- data/lib/rimless/configuration_handling.rb +11 -1
- data/lib/rimless/consumer.rb +4 -2
- data/lib/rimless/dependencies.rb +3 -0
- data/lib/rimless/kafka_helpers.rb +2 -0
- data/lib/rimless/karafka/avro_deserializer.rb +3 -3
- data/lib/rimless/rspec/helpers.rb +11 -0
- data/lib/rimless/rspec/matchers.rb +24 -9
- data/lib/rimless/rspec.rb +1 -1
- data/lib/rimless/tasks/consumer.rake +3 -0
- data/lib/rimless/tasks/generator.rake +3 -0
- data/lib/rimless/tasks/stats.rake +5 -2
- data/lib/rimless/version.rb +18 -1
- data/lib/rimless.rb +0 -1
- data/rimless.gemspec +43 -29
- metadata +121 -71
- data/.travis.yml +0 -35
- data/gemfiles/rails_4.2.gemfile +0 -8
- data/gemfiles/rails_5.0.gemfile +0 -8
- data/gemfiles/rails_5.1.gemfile +0 -8
- data/gemfiles/rails_6.0.gemfile +0 -8
@@ -4,6 +4,8 @@ module Rimless
|
|
4
4
|
# The top-level configuration handling.
|
5
5
|
#
|
6
6
|
# rubocop:disable Style/ClassVars because we split module code
|
7
|
+
# rubocop:disable Metrics/BlockLength because this is how
|
8
|
+
# an +ActiveSupport::Concern+ looks like
|
7
9
|
module ConfigurationHandling
|
8
10
|
extend ActiveSupport::Concern
|
9
11
|
|
@@ -61,8 +63,15 @@ module Rimless
|
|
61
63
|
# Check if a application is defined
|
62
64
|
return if Rails.application.nil?
|
63
65
|
|
66
|
+
# We need a little compatibility here, as in Rails 6.1+ there is no
|
67
|
+
# more +parent_name+, instead they renamed it to +module_parent_name+
|
68
|
+
app_class = Rails.application.class
|
69
|
+
parent_name = app_class.module_parent_name \
|
70
|
+
if app_class.respond_to?(:module_parent_name)
|
71
|
+
parent_name ||= app_class.parent_name
|
72
|
+
|
64
73
|
# Pass back the URI compatible application name
|
65
|
-
|
74
|
+
parent_name.underscore.dasherize
|
66
75
|
end
|
67
76
|
|
68
77
|
# Retrieve the current configured logger instance.
|
@@ -72,4 +81,5 @@ module Rimless
|
|
72
81
|
end
|
73
82
|
end
|
74
83
|
# rubocop:enable Style/ClassVars
|
84
|
+
# rubocop:enable Metrics/BlockLength
|
75
85
|
end
|
data/lib/rimless/consumer.rb
CHANGED
@@ -47,7 +47,7 @@ module Rimless
|
|
47
47
|
return unless rails_env.exist?
|
48
48
|
|
49
49
|
ENV['RAILS_ENV'] ||= 'development'
|
50
|
-
ENV['KARAFKA_ENV'] = ENV
|
50
|
+
ENV['KARAFKA_ENV'] = ENV.fetch('RAILS_ENV', nil)
|
51
51
|
require rails_env
|
52
52
|
Rails.application.eager_load!
|
53
53
|
end
|
@@ -67,6 +67,7 @@ module Rimless
|
|
67
67
|
# Configure the pure basics on the Karafka application.
|
68
68
|
#
|
69
69
|
# rubocop:disable Metrics/MethodLength because of the various settings
|
70
|
+
# rubocop:disable Metrics/AbcSize dito
|
70
71
|
def initialize_karafka!
|
71
72
|
setup do |config|
|
72
73
|
mapper = Rimless::Karafka::PassthroughMapper.new
|
@@ -82,13 +83,14 @@ module Rimless
|
|
82
83
|
end
|
83
84
|
end
|
84
85
|
# rubocop:enable Metrics/MethodLength
|
86
|
+
# rubocop:enable Metrics/AbcSize
|
85
87
|
|
86
88
|
# When we run in development mode, we want to write the logs
|
87
89
|
# to file and to stdout.
|
88
90
|
def initialize_logger!
|
89
91
|
return unless Rimless.env.development? && server?
|
90
92
|
|
91
|
-
|
93
|
+
$stdout.sync = true
|
92
94
|
Rimless.logger.extend(ActiveSupport::Logger.broadcast(
|
93
95
|
ActiveSupport::Logger.new($stdout)
|
94
96
|
))
|
data/lib/rimless/dependencies.rb
CHANGED
@@ -17,6 +17,8 @@ module Rimless
|
|
17
17
|
end
|
18
18
|
|
19
19
|
# Set sensible defaults for the +WaterDrop+ gem.
|
20
|
+
#
|
21
|
+
# rubocop:disable Metrics/AbcSize because of the configuration mapping
|
20
22
|
def configure_waterdrop
|
21
23
|
# Skip WaterDrop configuration when no brokers/client id is available,
|
22
24
|
# because it will raise. Its fine to have none available for situations
|
@@ -43,6 +45,7 @@ module Rimless
|
|
43
45
|
config.kafka.required_acks = -1
|
44
46
|
end
|
45
47
|
end
|
48
|
+
# rubocop:enable Metrics/AbcSize
|
46
49
|
|
47
50
|
# Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
|
48
51
|
# Avro schema templates (ERB), so the gem can handle them properly.
|
@@ -21,6 +21,7 @@ module Rimless
|
|
21
21
|
# Rimless.topic(name: 'test', app: :fancy_app)
|
22
22
|
#
|
23
23
|
# rubocop:disable Metrics/AbcSize because of the usage flexibility
|
24
|
+
# rubocop:disable Metrics/CyclomaticComplexity dito
|
24
25
|
def topic(*args)
|
25
26
|
opts = args.last
|
26
27
|
name = args.first if [String, Symbol].member?(args.first.class)
|
@@ -38,6 +39,7 @@ module Rimless
|
|
38
39
|
"#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
|
39
40
|
end
|
40
41
|
# rubocop:enable Metrics/AbcSize
|
42
|
+
# rubocop:enable Metrics/CyclomaticComplexity
|
41
43
|
|
42
44
|
# Send a single message to Apache Kafka. The data is encoded according to
|
43
45
|
# the given Apache Avro schema. The destination Kafka topic may be a
|
@@ -19,9 +19,9 @@ module Rimless
|
|
19
19
|
# occurence should be rare.
|
20
20
|
Rimless
|
21
21
|
.decode(params.raw_payload)
|
22
|
-
.
|
23
|
-
.
|
24
|
-
.
|
22
|
+
.then { |data| Sparsify(data, sparse_array: true) }
|
23
|
+
.then { |data| data.transform_keys { |key| key.delete('\\') } }
|
24
|
+
.then { |data| Unsparsify(data, sparse_array: true) }
|
25
25
|
.deep_symbolize_keys
|
26
26
|
end
|
27
27
|
end
|
@@ -23,6 +23,8 @@ module Rimless
|
|
23
23
|
# @return [OpenStruct] the fake deserialized Kafka message
|
24
24
|
#
|
25
25
|
# rubocop:disable Metrics/MethodLength because of the various properties
|
26
|
+
# rubocop:disable Style/OpenStructUse because existing specs may rely
|
27
|
+
# on this data type
|
26
28
|
def kafka_message(topic: nil, headers: {}, **payload)
|
27
29
|
OpenStruct.new(
|
28
30
|
topic: Rimless.topic(topic),
|
@@ -38,6 +40,15 @@ module Rimless
|
|
38
40
|
)
|
39
41
|
end
|
40
42
|
# rubocop:enable Metrics/MethodLength
|
43
|
+
# rubocop:enable Style/OpenStructUse
|
44
|
+
|
45
|
+
# Capture all Apache Kafka messages of the given block.
|
46
|
+
#
|
47
|
+
# @yield the given block to capture the messages
|
48
|
+
# @return [Array<Hash{Symbol => Mixed}>] the captured messages
|
49
|
+
def capture_kafka_messages(&block)
|
50
|
+
Rimless::RSpec::Matchers::HaveSentKafkaMessage.new(nil).capture(&block)
|
51
|
+
end
|
41
52
|
end
|
42
53
|
end
|
43
54
|
end
|
@@ -16,6 +16,7 @@ module Rimless
|
|
16
16
|
# @param schema [String, Symbol, nil] the expected message schema
|
17
17
|
# @return [HaveSentKafkaMessage] the expectation instance
|
18
18
|
def initialize(schema)
|
19
|
+
super
|
19
20
|
@schema = schema
|
20
21
|
@args = {}
|
21
22
|
@data = {}
|
@@ -23,6 +24,15 @@ module Rimless
|
|
23
24
|
set_expected_number(:exactly, 1)
|
24
25
|
end
|
25
26
|
|
27
|
+
# Capture all Apache Kafka messages of the given block.
|
28
|
+
#
|
29
|
+
# @yield the given block to capture the messages
|
30
|
+
# @return [Array<Hash{Symbol => Mixed}>] the captured messages
|
31
|
+
def capture(&block)
|
32
|
+
matches?(block)
|
33
|
+
@messages
|
34
|
+
end
|
35
|
+
|
26
36
|
# Collect the expectation arguments for the Kafka message passing. (eg.
|
27
37
|
# topic)
|
28
38
|
#
|
@@ -173,8 +183,9 @@ module Rimless
|
|
173
183
|
return true unless @schema
|
174
184
|
|
175
185
|
begin
|
176
|
-
Rimless.avro.decode(message[:
|
177
|
-
|
186
|
+
Rimless.avro.decode(message[:encoded_data],
|
187
|
+
schema_name: @schema.to_s)
|
188
|
+
true
|
178
189
|
rescue Avro::IO::SchemaMatchException
|
179
190
|
false
|
180
191
|
end
|
@@ -199,24 +210,28 @@ module Rimless
|
|
199
210
|
def data_match?(message)
|
200
211
|
return true unless @data.any?
|
201
212
|
|
202
|
-
|
203
|
-
expected_data = @data.deep_stringify_keys
|
204
|
-
|
205
|
-
actual_data.merge(expected_data) == actual_data
|
213
|
+
message[:data].merge(@data.deep_stringify_keys) == message[:data]
|
206
214
|
end
|
207
215
|
|
208
216
|
# Setup the +WaterDrop+ spies and record each sent message.
|
217
|
+
# because of the message decoding
|
218
|
+
# rubocop:disable Metrics/MethodLength dito
|
209
219
|
def listen_to_messages
|
220
|
+
decode = proc do |encoded|
|
221
|
+
{ encoded_data: encoded, data: Rimless.avro.decode(encoded) }
|
222
|
+
end
|
223
|
+
|
210
224
|
allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
|
211
|
-
@messages << {
|
225
|
+
@messages << { args: args, type: :sync }.merge(decode[data])
|
212
226
|
nil
|
213
227
|
end
|
214
228
|
|
215
229
|
allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
|
216
|
-
@messages << {
|
230
|
+
@messages << { args: args, type: :async }.merge(decode[data])
|
217
231
|
nil
|
218
232
|
end
|
219
233
|
end
|
234
|
+
# rubocop:enable Metrics/MethodLength
|
220
235
|
|
221
236
|
# Serve the RSpec API and return the positive failure message.
|
222
237
|
#
|
@@ -264,7 +279,7 @@ module Rimless
|
|
264
279
|
result = ['message']
|
265
280
|
|
266
281
|
result << " with #{message[:args]}" if message[:args].any?
|
267
|
-
result << " with data: #{
|
282
|
+
result << " with data: #{message[:data]}"
|
268
283
|
|
269
284
|
result.join
|
270
285
|
end
|
data/lib/rimless/rspec.rb
CHANGED
@@ -33,7 +33,7 @@ RSPEC_CONFIGURER.configure do |config|
|
|
33
33
|
# schema repository it cannot conflict while refreshing it.
|
34
34
|
unless ENV['TEST_ENV_NUMBER'].nil?
|
35
35
|
Rimless.configure do |conf|
|
36
|
-
num = ENV
|
36
|
+
num = ENV.fetch('TEST_ENV_NUMBER', nil)
|
37
37
|
num = '1' if num.empty?
|
38
38
|
|
39
39
|
conf.compiled_avro_schema_path = \
|
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
namespace :rimless do
|
4
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an command
|
5
|
+
# proxy, no need for an application bootstrap
|
4
6
|
desc 'Start the Apache Kafka consumer'
|
5
7
|
task :consumer do
|
6
8
|
system 'bundle exec karafka server'
|
7
9
|
end
|
10
|
+
# rubocop:enable Rails/RakeEnvironment
|
8
11
|
|
9
12
|
desc 'Print all the consumer routes'
|
10
13
|
task routes: :environment do
|
@@ -18,6 +18,8 @@ namespace :rimless do
|
|
18
18
|
FileUtils.copy(src, dest)
|
19
19
|
end
|
20
20
|
|
21
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an
|
22
|
+
# helper command, no need for an application bootstrap
|
21
23
|
desc 'Install the Rimless consumer components'
|
22
24
|
task :install do
|
23
25
|
install_template('karafka.rb')
|
@@ -33,4 +35,5 @@ namespace :rimless do
|
|
33
35
|
# your project root. And list all routes with +rake rimless:routes+.
|
34
36
|
OUTPUT
|
35
37
|
end
|
38
|
+
# rubocop:enable Rails/RakeEnvironment
|
36
39
|
end
|
@@ -3,6 +3,8 @@
|
|
3
3
|
if defined?(Rails) && Rails.env.development?
|
4
4
|
require 'rspec/core/rake_task'
|
5
5
|
|
6
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an
|
7
|
+
# helper command, no need for an application bootstrap
|
6
8
|
task :stats do
|
7
9
|
require 'rails/code_statistics'
|
8
10
|
|
@@ -11,8 +13,9 @@ if defined?(Rails) && Rails.env.development?
|
|
11
13
|
].each do |method, type, dir|
|
12
14
|
next unless File.directory? dir
|
13
15
|
|
14
|
-
|
15
|
-
|
16
|
+
STATS_DIRECTORIES.send(method, [type, dir])
|
17
|
+
CodeStatistics::TEST_TYPES << type if type.include? 'specs'
|
16
18
|
end
|
17
19
|
end
|
20
|
+
# rubocop:enable Rails/RakeEnvironment
|
18
21
|
end
|
data/lib/rimless/version.rb
CHANGED
@@ -1,6 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# The gem version details.
|
3
4
|
module Rimless
|
4
5
|
# The version of the +rimless+ gem
|
5
|
-
VERSION = '1.
|
6
|
+
VERSION = '1.3.0'
|
7
|
+
|
8
|
+
class << self
|
9
|
+
# Returns the version of gem as a string.
|
10
|
+
#
|
11
|
+
# @return [String] the gem version as string
|
12
|
+
def version
|
13
|
+
VERSION
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns the version of the gem as a +Gem::Version+.
|
17
|
+
#
|
18
|
+
# @return [Gem::Version] the gem version as object
|
19
|
+
def gem_version
|
20
|
+
Gem::Version.new VERSION
|
21
|
+
end
|
22
|
+
end
|
6
23
|
end
|
data/lib/rimless.rb
CHANGED
data/rimless.gemspec
CHANGED
@@ -5,47 +5,61 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
require 'rimless/version'
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.name
|
9
|
-
spec.version
|
10
|
-
spec.authors
|
11
|
-
spec.email
|
8
|
+
spec.name = 'rimless'
|
9
|
+
spec.version = Rimless::VERSION
|
10
|
+
spec.authors = ['Hermann Mayer']
|
11
|
+
spec.email = ['hermann.mayer92@gmail.com']
|
12
12
|
|
13
|
-
spec.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
spec.license = 'MIT'
|
14
|
+
spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
|
15
|
+
'Schema Registry helpers.'
|
16
|
+
spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
|
17
|
+
'Schema Registry helpers.'
|
18
|
+
|
19
|
+
base_uri = "https://github.com/hausgold/#{spec.name}"
|
20
|
+
spec.metadata = {
|
21
|
+
'homepage_uri' => base_uri,
|
22
|
+
'source_code_uri' => base_uri,
|
23
|
+
'changelog_uri' => "#{base_uri}/blob/master/CHANGELOG.md",
|
24
|
+
'bug_tracker_uri' => "#{base_uri}/issues",
|
25
|
+
'documentation_uri' => "https://www.rubydoc.info/gems/#{spec.name}"
|
26
|
+
}
|
18
27
|
|
19
28
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
20
29
|
f.match(%r{^(test|spec|features)/})
|
21
30
|
end
|
22
|
-
|
23
|
-
spec.
|
31
|
+
|
32
|
+
spec.bindir = 'exe'
|
33
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
34
|
spec.require_paths = ['lib']
|
25
35
|
|
26
|
-
spec.
|
36
|
+
spec.required_ruby_version = '>= 2.5'
|
37
|
+
|
38
|
+
spec.add_runtime_dependency 'activesupport', '>= 5.2'
|
27
39
|
spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
|
28
40
|
spec.add_runtime_dependency 'karafka', '~> 1.4'
|
29
41
|
spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.4'
|
30
42
|
spec.add_runtime_dependency 'karafka-testing', '~> 1.4'
|
31
|
-
spec.add_runtime_dependency 'sinatra'
|
43
|
+
spec.add_runtime_dependency 'sinatra', '~> 2.2'
|
32
44
|
spec.add_runtime_dependency 'sparsify', '~> 1.1'
|
33
|
-
spec.add_runtime_dependency 'waterdrop', '~> 1.
|
34
|
-
spec.add_runtime_dependency 'webmock'
|
45
|
+
spec.add_runtime_dependency 'waterdrop', '~> 1.4'
|
46
|
+
spec.add_runtime_dependency 'webmock', '~> 3.18'
|
35
47
|
|
36
|
-
spec.add_development_dependency 'appraisal'
|
37
|
-
spec.add_development_dependency 'bundler', '
|
38
|
-
spec.add_development_dependency '
|
39
|
-
spec.add_development_dependency '
|
48
|
+
spec.add_development_dependency 'appraisal', '~> 2.4'
|
49
|
+
spec.add_development_dependency 'bundler', '~> 2.3'
|
50
|
+
spec.add_development_dependency 'countless', '~> 1.1'
|
51
|
+
spec.add_development_dependency 'factory_bot', '~> 6.2'
|
52
|
+
spec.add_development_dependency 'guard-rspec', '~> 4.7'
|
53
|
+
spec.add_development_dependency 'railties', '>= 5.2'
|
40
54
|
spec.add_development_dependency 'rake', '~> 13.0'
|
41
|
-
spec.add_development_dependency '
|
42
|
-
spec.add_development_dependency '
|
43
|
-
spec.add_development_dependency '
|
44
|
-
spec.add_development_dependency 'rubocop', '~>
|
45
|
-
spec.add_development_dependency 'rubocop-rspec', '~>
|
46
|
-
spec.add_development_dependency 'simplecov', '
|
47
|
-
spec.add_development_dependency 'timecop', '
|
48
|
-
spec.add_development_dependency 'vcr', '~>
|
49
|
-
spec.add_development_dependency 'yard', '
|
50
|
-
spec.add_development_dependency 'yard-activesupport-concern', '
|
55
|
+
spec.add_development_dependency 'redcarpet', '~> 3.5'
|
56
|
+
spec.add_development_dependency 'rspec', '~> 3.12'
|
57
|
+
spec.add_development_dependency 'rubocop', '~> 1.28'
|
58
|
+
spec.add_development_dependency 'rubocop-rails', '~> 2.14'
|
59
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 2.10'
|
60
|
+
spec.add_development_dependency 'simplecov', '>= 0.22'
|
61
|
+
spec.add_development_dependency 'timecop', '>= 0.9.6'
|
62
|
+
spec.add_development_dependency 'vcr', '~> 6.0'
|
63
|
+
spec.add_development_dependency 'yard', '>= 0.9.28'
|
64
|
+
spec.add_development_dependency 'yard-activesupport-concern', '>= 0.0.1'
|
51
65
|
end
|