rimless 1.1.1 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/documentation.yml +39 -0
- data/.github/workflows/test.yml +63 -0
- data/.rspec +2 -2
- data/.rubocop.yml +16 -2
- data/.simplecov +12 -0
- data/Appraisals +2 -22
- data/CHANGELOG.md +12 -0
- data/Dockerfile +2 -3
- data/Envfile +0 -3
- data/Guardfile +44 -0
- data/LICENSE +1 -1
- data/Makefile +25 -15
- data/README.md +11 -4
- data/Rakefile +13 -65
- data/doc/kafka-playground/.gitignore +2 -0
- data/doc/kafka-playground/Dockerfile +41 -0
- data/doc/kafka-playground/Gemfile +8 -0
- data/doc/kafka-playground/Gemfile.lock +155 -0
- data/doc/kafka-playground/Makefile +209 -0
- data/doc/kafka-playground/README.md +185 -0
- data/doc/kafka-playground/bin/consume-topic +7 -0
- data/doc/kafka-playground/bin/create-topic +42 -0
- data/doc/kafka-playground/bin/delete-topic +22 -0
- data/doc/kafka-playground/bin/list-topics +3 -0
- data/doc/kafka-playground/bin/produce-event +64 -0
- data/doc/kafka-playground/config/avro_schemas/.gitignore +1 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/item_v1.avsc.erb +36 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1.avsc.erb +59 -0
- data/doc/kafka-playground/config/avro_schemas/playground_app/payment_v1_event.avsc.erb +18 -0
- data/doc/kafka-playground/config/docker/shell/.bash_profile +3 -0
- data/doc/kafka-playground/config/docker/shell/.bashrc +231 -0
- data/doc/kafka-playground/config/docker/shell/.config/kcat.conf +3 -0
- data/doc/kafka-playground/config/docker/shell/.gemrc +2 -0
- data/doc/kafka-playground/config/docker/shell/.inputrc +17 -0
- data/doc/kafka-playground/config/environment.rb +69 -0
- data/doc/kafka-playground/doc/assets/project.svg +68 -0
- data/doc/kafka-playground/docker-compose.yml +83 -0
- data/doc/kafka-playground/examples/rimless-produce +48 -0
- data/gemfiles/rails_5.2.gemfile +2 -2
- data/lib/rimless/configuration_handling.rb +11 -1
- data/lib/rimless/consumer.rb +4 -2
- data/lib/rimless/dependencies.rb +3 -0
- data/lib/rimless/kafka_helpers.rb +2 -0
- data/lib/rimless/karafka/avro_deserializer.rb +3 -3
- data/lib/rimless/rspec/helpers.rb +11 -0
- data/lib/rimless/rspec/matchers.rb +24 -9
- data/lib/rimless/rspec.rb +1 -1
- data/lib/rimless/tasks/consumer.rake +3 -0
- data/lib/rimless/tasks/generator.rake +3 -0
- data/lib/rimless/tasks/stats.rake +5 -2
- data/lib/rimless/version.rb +18 -1
- data/lib/rimless.rb +0 -1
- data/rimless.gemspec +43 -29
- metadata +121 -71
- data/.travis.yml +0 -35
- data/gemfiles/rails_4.2.gemfile +0 -8
- data/gemfiles/rails_5.0.gemfile +0 -8
- data/gemfiles/rails_5.1.gemfile +0 -8
- data/gemfiles/rails_6.0.gemfile +0 -8
@@ -4,6 +4,8 @@ module Rimless
|
|
4
4
|
# The top-level configuration handling.
|
5
5
|
#
|
6
6
|
# rubocop:disable Style/ClassVars because we split module code
|
7
|
+
# rubocop:disable Metrics/BlockLength because this is how
|
8
|
+
# an +ActiveSupport::Concern+ looks like
|
7
9
|
module ConfigurationHandling
|
8
10
|
extend ActiveSupport::Concern
|
9
11
|
|
@@ -61,8 +63,15 @@ module Rimless
|
|
61
63
|
# Check if a application is defined
|
62
64
|
return if Rails.application.nil?
|
63
65
|
|
66
|
+
# We need a little compatibility here, as in Rails 6.1+ there is no
|
67
|
+
# more +parent_name+, instead they renamed it to +module_parent_name+
|
68
|
+
app_class = Rails.application.class
|
69
|
+
parent_name = app_class.module_parent_name \
|
70
|
+
if app_class.respond_to?(:module_parent_name)
|
71
|
+
parent_name ||= app_class.parent_name
|
72
|
+
|
64
73
|
# Pass back the URI compatible application name
|
65
|
-
|
74
|
+
parent_name.underscore.dasherize
|
66
75
|
end
|
67
76
|
|
68
77
|
# Retrieve the current configured logger instance.
|
@@ -72,4 +81,5 @@ module Rimless
|
|
72
81
|
end
|
73
82
|
end
|
74
83
|
# rubocop:enable Style/ClassVars
|
84
|
+
# rubocop:enable Metrics/BlockLength
|
75
85
|
end
|
data/lib/rimless/consumer.rb
CHANGED
@@ -47,7 +47,7 @@ module Rimless
|
|
47
47
|
return unless rails_env.exist?
|
48
48
|
|
49
49
|
ENV['RAILS_ENV'] ||= 'development'
|
50
|
-
ENV['KARAFKA_ENV'] = ENV
|
50
|
+
ENV['KARAFKA_ENV'] = ENV.fetch('RAILS_ENV', nil)
|
51
51
|
require rails_env
|
52
52
|
Rails.application.eager_load!
|
53
53
|
end
|
@@ -67,6 +67,7 @@ module Rimless
|
|
67
67
|
# Configure the pure basics on the Karafka application.
|
68
68
|
#
|
69
69
|
# rubocop:disable Metrics/MethodLength because of the various settings
|
70
|
+
# rubocop:disable Metrics/AbcSize dito
|
70
71
|
def initialize_karafka!
|
71
72
|
setup do |config|
|
72
73
|
mapper = Rimless::Karafka::PassthroughMapper.new
|
@@ -82,13 +83,14 @@ module Rimless
|
|
82
83
|
end
|
83
84
|
end
|
84
85
|
# rubocop:enable Metrics/MethodLength
|
86
|
+
# rubocop:enable Metrics/AbcSize
|
85
87
|
|
86
88
|
# When we run in development mode, we want to write the logs
|
87
89
|
# to file and to stdout.
|
88
90
|
def initialize_logger!
|
89
91
|
return unless Rimless.env.development? && server?
|
90
92
|
|
91
|
-
|
93
|
+
$stdout.sync = true
|
92
94
|
Rimless.logger.extend(ActiveSupport::Logger.broadcast(
|
93
95
|
ActiveSupport::Logger.new($stdout)
|
94
96
|
))
|
data/lib/rimless/dependencies.rb
CHANGED
@@ -17,6 +17,8 @@ module Rimless
|
|
17
17
|
end
|
18
18
|
|
19
19
|
# Set sensible defaults for the +WaterDrop+ gem.
|
20
|
+
#
|
21
|
+
# rubocop:disable Metrics/AbcSize because of the configuration mapping
|
20
22
|
def configure_waterdrop
|
21
23
|
# Skip WaterDrop configuration when no brokers/client id is available,
|
22
24
|
# because it will raise. Its fine to have none available for situations
|
@@ -43,6 +45,7 @@ module Rimless
|
|
43
45
|
config.kafka.required_acks = -1
|
44
46
|
end
|
45
47
|
end
|
48
|
+
# rubocop:enable Metrics/AbcSize
|
46
49
|
|
47
50
|
# Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
|
48
51
|
# Avro schema templates (ERB), so the gem can handle them properly.
|
@@ -21,6 +21,7 @@ module Rimless
|
|
21
21
|
# Rimless.topic(name: 'test', app: :fancy_app)
|
22
22
|
#
|
23
23
|
# rubocop:disable Metrics/AbcSize because of the usage flexibility
|
24
|
+
# rubocop:disable Metrics/CyclomaticComplexity dito
|
24
25
|
def topic(*args)
|
25
26
|
opts = args.last
|
26
27
|
name = args.first if [String, Symbol].member?(args.first.class)
|
@@ -38,6 +39,7 @@ module Rimless
|
|
38
39
|
"#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
|
39
40
|
end
|
40
41
|
# rubocop:enable Metrics/AbcSize
|
42
|
+
# rubocop:enable Metrics/CyclomaticComplexity
|
41
43
|
|
42
44
|
# Send a single message to Apache Kafka. The data is encoded according to
|
43
45
|
# the given Apache Avro schema. The destination Kafka topic may be a
|
@@ -19,9 +19,9 @@ module Rimless
|
|
19
19
|
# occurence should be rare.
|
20
20
|
Rimless
|
21
21
|
.decode(params.raw_payload)
|
22
|
-
.
|
23
|
-
.
|
24
|
-
.
|
22
|
+
.then { |data| Sparsify(data, sparse_array: true) }
|
23
|
+
.then { |data| data.transform_keys { |key| key.delete('\\') } }
|
24
|
+
.then { |data| Unsparsify(data, sparse_array: true) }
|
25
25
|
.deep_symbolize_keys
|
26
26
|
end
|
27
27
|
end
|
@@ -23,6 +23,8 @@ module Rimless
|
|
23
23
|
# @return [OpenStruct] the fake deserialized Kafka message
|
24
24
|
#
|
25
25
|
# rubocop:disable Metrics/MethodLength because of the various properties
|
26
|
+
# rubocop:disable Style/OpenStructUse because existing specs may rely
|
27
|
+
# on this data type
|
26
28
|
def kafka_message(topic: nil, headers: {}, **payload)
|
27
29
|
OpenStruct.new(
|
28
30
|
topic: Rimless.topic(topic),
|
@@ -38,6 +40,15 @@ module Rimless
|
|
38
40
|
)
|
39
41
|
end
|
40
42
|
# rubocop:enable Metrics/MethodLength
|
43
|
+
# rubocop:enable Style/OpenStructUse
|
44
|
+
|
45
|
+
# Capture all Apache Kafka messages of the given block.
|
46
|
+
#
|
47
|
+
# @yield the given block to capture the messages
|
48
|
+
# @return [Array<Hash{Symbol => Mixed}>] the captured messages
|
49
|
+
def capture_kafka_messages(&block)
|
50
|
+
Rimless::RSpec::Matchers::HaveSentKafkaMessage.new(nil).capture(&block)
|
51
|
+
end
|
41
52
|
end
|
42
53
|
end
|
43
54
|
end
|
@@ -16,6 +16,7 @@ module Rimless
|
|
16
16
|
# @param schema [String, Symbol, nil] the expected message schema
|
17
17
|
# @return [HaveSentKafkaMessage] the expectation instance
|
18
18
|
def initialize(schema)
|
19
|
+
super
|
19
20
|
@schema = schema
|
20
21
|
@args = {}
|
21
22
|
@data = {}
|
@@ -23,6 +24,15 @@ module Rimless
|
|
23
24
|
set_expected_number(:exactly, 1)
|
24
25
|
end
|
25
26
|
|
27
|
+
# Capture all Apache Kafka messages of the given block.
|
28
|
+
#
|
29
|
+
# @yield the given block to capture the messages
|
30
|
+
# @return [Array<Hash{Symbol => Mixed}>] the captured messages
|
31
|
+
def capture(&block)
|
32
|
+
matches?(block)
|
33
|
+
@messages
|
34
|
+
end
|
35
|
+
|
26
36
|
# Collect the expectation arguments for the Kafka message passing. (eg.
|
27
37
|
# topic)
|
28
38
|
#
|
@@ -173,8 +183,9 @@ module Rimless
|
|
173
183
|
return true unless @schema
|
174
184
|
|
175
185
|
begin
|
176
|
-
Rimless.avro.decode(message[:
|
177
|
-
|
186
|
+
Rimless.avro.decode(message[:encoded_data],
|
187
|
+
schema_name: @schema.to_s)
|
188
|
+
true
|
178
189
|
rescue Avro::IO::SchemaMatchException
|
179
190
|
false
|
180
191
|
end
|
@@ -199,24 +210,28 @@ module Rimless
|
|
199
210
|
def data_match?(message)
|
200
211
|
return true unless @data.any?
|
201
212
|
|
202
|
-
|
203
|
-
expected_data = @data.deep_stringify_keys
|
204
|
-
|
205
|
-
actual_data.merge(expected_data) == actual_data
|
213
|
+
message[:data].merge(@data.deep_stringify_keys) == message[:data]
|
206
214
|
end
|
207
215
|
|
208
216
|
# Setup the +WaterDrop+ spies and record each sent message.
|
217
|
+
# because of the message decoding
|
218
|
+
# rubocop:disable Metrics/MethodLength dito
|
209
219
|
def listen_to_messages
|
220
|
+
decode = proc do |encoded|
|
221
|
+
{ encoded_data: encoded, data: Rimless.avro.decode(encoded) }
|
222
|
+
end
|
223
|
+
|
210
224
|
allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
|
211
|
-
@messages << {
|
225
|
+
@messages << { args: args, type: :sync }.merge(decode[data])
|
212
226
|
nil
|
213
227
|
end
|
214
228
|
|
215
229
|
allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
|
216
|
-
@messages << {
|
230
|
+
@messages << { args: args, type: :async }.merge(decode[data])
|
217
231
|
nil
|
218
232
|
end
|
219
233
|
end
|
234
|
+
# rubocop:enable Metrics/MethodLength
|
220
235
|
|
221
236
|
# Serve the RSpec API and return the positive failure message.
|
222
237
|
#
|
@@ -264,7 +279,7 @@ module Rimless
|
|
264
279
|
result = ['message']
|
265
280
|
|
266
281
|
result << " with #{message[:args]}" if message[:args].any?
|
267
|
-
result << " with data: #{
|
282
|
+
result << " with data: #{message[:data]}"
|
268
283
|
|
269
284
|
result.join
|
270
285
|
end
|
data/lib/rimless/rspec.rb
CHANGED
@@ -33,7 +33,7 @@ RSPEC_CONFIGURER.configure do |config|
|
|
33
33
|
# schema repository it cannot conflict while refreshing it.
|
34
34
|
unless ENV['TEST_ENV_NUMBER'].nil?
|
35
35
|
Rimless.configure do |conf|
|
36
|
-
num = ENV
|
36
|
+
num = ENV.fetch('TEST_ENV_NUMBER', nil)
|
37
37
|
num = '1' if num.empty?
|
38
38
|
|
39
39
|
conf.compiled_avro_schema_path = \
|
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
namespace :rimless do
|
4
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an command
|
5
|
+
# proxy, no need for an application bootstrap
|
4
6
|
desc 'Start the Apache Kafka consumer'
|
5
7
|
task :consumer do
|
6
8
|
system 'bundle exec karafka server'
|
7
9
|
end
|
10
|
+
# rubocop:enable Rails/RakeEnvironment
|
8
11
|
|
9
12
|
desc 'Print all the consumer routes'
|
10
13
|
task routes: :environment do
|
@@ -18,6 +18,8 @@ namespace :rimless do
|
|
18
18
|
FileUtils.copy(src, dest)
|
19
19
|
end
|
20
20
|
|
21
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an
|
22
|
+
# helper command, no need for an application bootstrap
|
21
23
|
desc 'Install the Rimless consumer components'
|
22
24
|
task :install do
|
23
25
|
install_template('karafka.rb')
|
@@ -33,4 +35,5 @@ namespace :rimless do
|
|
33
35
|
# your project root. And list all routes with +rake rimless:routes+.
|
34
36
|
OUTPUT
|
35
37
|
end
|
38
|
+
# rubocop:enable Rails/RakeEnvironment
|
36
39
|
end
|
@@ -3,6 +3,8 @@
|
|
3
3
|
if defined?(Rails) && Rails.env.development?
|
4
4
|
require 'rspec/core/rake_task'
|
5
5
|
|
6
|
+
# rubocop:disable Rails/RakeEnvironment because this is just an
|
7
|
+
# helper command, no need for an application bootstrap
|
6
8
|
task :stats do
|
7
9
|
require 'rails/code_statistics'
|
8
10
|
|
@@ -11,8 +13,9 @@ if defined?(Rails) && Rails.env.development?
|
|
11
13
|
].each do |method, type, dir|
|
12
14
|
next unless File.directory? dir
|
13
15
|
|
14
|
-
|
15
|
-
|
16
|
+
STATS_DIRECTORIES.send(method, [type, dir])
|
17
|
+
CodeStatistics::TEST_TYPES << type if type.include? 'specs'
|
16
18
|
end
|
17
19
|
end
|
20
|
+
# rubocop:enable Rails/RakeEnvironment
|
18
21
|
end
|
data/lib/rimless/version.rb
CHANGED
@@ -1,6 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# The gem version details.
|
3
4
|
module Rimless
|
4
5
|
# The version of the +rimless+ gem
|
5
|
-
VERSION = '1.
|
6
|
+
VERSION = '1.3.0'
|
7
|
+
|
8
|
+
class << self
|
9
|
+
# Returns the version of gem as a string.
|
10
|
+
#
|
11
|
+
# @return [String] the gem version as string
|
12
|
+
def version
|
13
|
+
VERSION
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns the version of the gem as a +Gem::Version+.
|
17
|
+
#
|
18
|
+
# @return [Gem::Version] the gem version as object
|
19
|
+
def gem_version
|
20
|
+
Gem::Version.new VERSION
|
21
|
+
end
|
22
|
+
end
|
6
23
|
end
|
data/lib/rimless.rb
CHANGED
data/rimless.gemspec
CHANGED
@@ -5,47 +5,61 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
require 'rimless/version'
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.name
|
9
|
-
spec.version
|
10
|
-
spec.authors
|
11
|
-
spec.email
|
8
|
+
spec.name = 'rimless'
|
9
|
+
spec.version = Rimless::VERSION
|
10
|
+
spec.authors = ['Hermann Mayer']
|
11
|
+
spec.email = ['hermann.mayer92@gmail.com']
|
12
12
|
|
13
|
-
spec.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
spec.license = 'MIT'
|
14
|
+
spec.summary = 'A bundle of opinionated Apache Kafka / Confluent ' \
|
15
|
+
'Schema Registry helpers.'
|
16
|
+
spec.description = 'A bundle of opinionated Apache Kafka / Confluent ' \
|
17
|
+
'Schema Registry helpers.'
|
18
|
+
|
19
|
+
base_uri = "https://github.com/hausgold/#{spec.name}"
|
20
|
+
spec.metadata = {
|
21
|
+
'homepage_uri' => base_uri,
|
22
|
+
'source_code_uri' => base_uri,
|
23
|
+
'changelog_uri' => "#{base_uri}/blob/master/CHANGELOG.md",
|
24
|
+
'bug_tracker_uri' => "#{base_uri}/issues",
|
25
|
+
'documentation_uri' => "https://www.rubydoc.info/gems/#{spec.name}"
|
26
|
+
}
|
18
27
|
|
19
28
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
20
29
|
f.match(%r{^(test|spec|features)/})
|
21
30
|
end
|
22
|
-
|
23
|
-
spec.
|
31
|
+
|
32
|
+
spec.bindir = 'exe'
|
33
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
34
|
spec.require_paths = ['lib']
|
25
35
|
|
26
|
-
spec.
|
36
|
+
spec.required_ruby_version = '>= 2.5'
|
37
|
+
|
38
|
+
spec.add_runtime_dependency 'activesupport', '>= 5.2'
|
27
39
|
spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
|
28
40
|
spec.add_runtime_dependency 'karafka', '~> 1.4'
|
29
41
|
spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.4'
|
30
42
|
spec.add_runtime_dependency 'karafka-testing', '~> 1.4'
|
31
|
-
spec.add_runtime_dependency 'sinatra'
|
43
|
+
spec.add_runtime_dependency 'sinatra', '~> 2.2'
|
32
44
|
spec.add_runtime_dependency 'sparsify', '~> 1.1'
|
33
|
-
spec.add_runtime_dependency 'waterdrop', '~> 1.
|
34
|
-
spec.add_runtime_dependency 'webmock'
|
45
|
+
spec.add_runtime_dependency 'waterdrop', '~> 1.4'
|
46
|
+
spec.add_runtime_dependency 'webmock', '~> 3.18'
|
35
47
|
|
36
|
-
spec.add_development_dependency 'appraisal'
|
37
|
-
spec.add_development_dependency 'bundler', '
|
38
|
-
spec.add_development_dependency '
|
39
|
-
spec.add_development_dependency '
|
48
|
+
spec.add_development_dependency 'appraisal', '~> 2.4'
|
49
|
+
spec.add_development_dependency 'bundler', '~> 2.3'
|
50
|
+
spec.add_development_dependency 'countless', '~> 1.1'
|
51
|
+
spec.add_development_dependency 'factory_bot', '~> 6.2'
|
52
|
+
spec.add_development_dependency 'guard-rspec', '~> 4.7'
|
53
|
+
spec.add_development_dependency 'railties', '>= 5.2'
|
40
54
|
spec.add_development_dependency 'rake', '~> 13.0'
|
41
|
-
spec.add_development_dependency '
|
42
|
-
spec.add_development_dependency '
|
43
|
-
spec.add_development_dependency '
|
44
|
-
spec.add_development_dependency 'rubocop', '~>
|
45
|
-
spec.add_development_dependency 'rubocop-rspec', '~>
|
46
|
-
spec.add_development_dependency 'simplecov', '
|
47
|
-
spec.add_development_dependency 'timecop', '
|
48
|
-
spec.add_development_dependency 'vcr', '~>
|
49
|
-
spec.add_development_dependency 'yard', '
|
50
|
-
spec.add_development_dependency 'yard-activesupport-concern', '
|
55
|
+
spec.add_development_dependency 'redcarpet', '~> 3.5'
|
56
|
+
spec.add_development_dependency 'rspec', '~> 3.12'
|
57
|
+
spec.add_development_dependency 'rubocop', '~> 1.28'
|
58
|
+
spec.add_development_dependency 'rubocop-rails', '~> 2.14'
|
59
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 2.10'
|
60
|
+
spec.add_development_dependency 'simplecov', '>= 0.22'
|
61
|
+
spec.add_development_dependency 'timecop', '>= 0.9.6'
|
62
|
+
spec.add_development_dependency 'vcr', '~> 6.0'
|
63
|
+
spec.add_development_dependency 'yard', '>= 0.9.28'
|
64
|
+
spec.add_development_dependency 'yard-activesupport-concern', '>= 0.0.1'
|
51
65
|
end
|