sbmt-kafka_consumer 2.8.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Appraisals +2 -2
- data/CHANGELOG.md +18 -0
- data/LICENSE +1 -1
- data/README.md +4 -4
- data/dip.yml +3 -3
- data/lib/sbmt/kafka_consumer/client_configurer.rb +5 -4
- data/lib/sbmt/kafka_consumer/routing/consumer_mapper/base.rb +17 -0
- data/lib/sbmt/kafka_consumer/routing/karafka_v1_consumer_mapper.rb +6 -1
- data/lib/sbmt/kafka_consumer/routing/karafka_v2_consumer_mapper.rb +10 -3
- data/lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb +11 -11
- data/lib/sbmt/kafka_consumer/version.rb +1 -1
- data/sbmt-kafka_consumer.gemspec +5 -5
- metadata +10 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f56ef789be2f19f5e5890292e8163f29d65ced1de23358b3a0bcf4163b77587d
|
|
4
|
+
data.tar.gz: 6f7450a9ecf65d6d32294f26142706a7c523af2073be90e9cb9bc6a630ae73f9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0bab92c52ee0dd45b2893f1e592a480b53c055d2addd522d8ab22bdc8b83068983734974ba56aaf47d370dd6f42b342ec5e1cc8623f8e8a4833ccf112e23bdf2
|
|
7
|
+
data.tar.gz: f159851f5089686e1b767f53852257d10575bb4c532e603f8d3a9169f7500556a802bf0e79e60de19c6056d213347782a7334f6a67c5f9ec5f3bc9a2085c472f
|
data/Appraisals
CHANGED
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
# See compatibility table at https://www.fastruby.io/blog/ruby/rails/versions/compatibility-table.html
|
|
4
4
|
|
|
5
5
|
versions_map = {
|
|
6
|
-
"6.0" => %w[2.7],
|
|
7
6
|
"6.1" => %w[2.7 3.0],
|
|
8
7
|
"7.0" => %w[3.1],
|
|
9
|
-
"7.1" => %w[3.2, 3.3]
|
|
8
|
+
"7.1" => %w[3.2, 3.3],
|
|
9
|
+
"7.2" => %w[3.3]
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
current_ruby_version = RUBY_VERSION.split(".").first(2).join(".")
|
data/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
|
13
13
|
|
|
14
14
|
### Fixed
|
|
15
15
|
|
|
16
|
+
## [3.1.0] - 2024-09-09
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- Refactor consumer class initialization
|
|
21
|
+
|
|
22
|
+
## [3.0.0] - 2024-09-04
|
|
23
|
+
|
|
24
|
+
## BREAKING
|
|
25
|
+
|
|
26
|
+
- Drop support for Ruby 2.7
|
|
27
|
+
- Drop support for Rails 6.0
|
|
28
|
+
- Add support for Karafka 2.4
|
|
29
|
+
|
|
30
|
+
### Fixed
|
|
31
|
+
|
|
32
|
+
- Support consumer group mappers to support backward compatibility of consumer group naming
|
|
33
|
+
|
|
16
34
|
## [2.8.0] - 2024-09-09
|
|
17
35
|
|
|
18
36
|
### Fixed
|
data/LICENSE
CHANGED
data/README.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
[](https://badge.fury.io/rb/sbmt-kafka_consumer)
|
|
2
|
-
[](https://github.com/Kuper-Tech/sbmt-kafka_consumer/actions?query=branch%3Amaster)
|
|
3
3
|
|
|
4
4
|
# Sbmt-KafkaConsumer
|
|
5
5
|
|
|
6
|
-
This gem is used to consume Kafka messages. It is a wrapper over the [Karafka](https://github.com/karafka/karafka) gem, and is recommended for use as a transport with the [sbmt-outbox](https://github.com/
|
|
6
|
+
This gem is used to consume Kafka messages. It is a wrapper over the [Karafka](https://github.com/karafka/karafka) gem, and is recommended for use as a transport with the [sbmt-outbox](https://github.com/Kuper-Tech/sbmt-outbox) gem.
|
|
7
7
|
|
|
8
8
|
## Installation
|
|
9
9
|
|
|
@@ -21,7 +21,7 @@ bundle install
|
|
|
21
21
|
|
|
22
22
|
## Demo
|
|
23
23
|
|
|
24
|
-
Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/
|
|
24
|
+
Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/Kuper-Tech/outbox-example-apps
|
|
25
25
|
|
|
26
26
|
## Auto configuration
|
|
27
27
|
|
|
@@ -47,7 +47,7 @@ rails g kafka_consumer:consumer MaybeNamespaced::Name
|
|
|
47
47
|
|
|
48
48
|
### Inbox consumer
|
|
49
49
|
|
|
50
|
-
To generate an Inbox consumer for use with gem [sbmt-outbox](https://github.com/
|
|
50
|
+
To generate an Inbox consumer for use with gem [sbmt-outbox](https://github.com/Kuper-Tech/sbmt-outbox), run the following command:
|
|
51
51
|
|
|
52
52
|
```shell
|
|
53
53
|
rails g kafka_consumer:inbox_consumer MaybeNamespaced::Name some-consumer-group some-topic
|
data/dip.yml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
version: '7'
|
|
2
2
|
|
|
3
3
|
environment:
|
|
4
|
-
RUBY_VERSION: '3.
|
|
4
|
+
RUBY_VERSION: '3.3'
|
|
5
5
|
|
|
6
6
|
compose:
|
|
7
7
|
files:
|
|
@@ -35,14 +35,14 @@ interaction:
|
|
|
35
35
|
subcommands:
|
|
36
36
|
all:
|
|
37
37
|
command: bundle exec appraisal rspec
|
|
38
|
-
rails-6.0:
|
|
39
|
-
command: bundle exec appraisal rails-6.0 rspec
|
|
40
38
|
rails-6.1:
|
|
41
39
|
command: bundle exec appraisal rails-6.1 rspec
|
|
42
40
|
rails-7.0:
|
|
43
41
|
command: bundle exec appraisal rails-7.0 rspec
|
|
44
42
|
rails-7.1:
|
|
45
43
|
command: bundle exec appraisal rails-7.1 rspec
|
|
44
|
+
rails-7.2:
|
|
45
|
+
command: bundle exec appraisal rails-7.2 rspec
|
|
46
46
|
|
|
47
47
|
rubocop:
|
|
48
48
|
description: Run Ruby linter
|
|
@@ -6,10 +6,8 @@ class Sbmt::KafkaConsumer::ClientConfigurer
|
|
|
6
6
|
Karafka::App.setup do |karafka_config|
|
|
7
7
|
karafka_config.monitor = config.monitor_class.classify.constantize.new
|
|
8
8
|
karafka_config.logger = Sbmt::KafkaConsumer.logger
|
|
9
|
-
karafka_config.deserializer = config.deserializer_class.classify.constantize.new
|
|
10
9
|
|
|
11
10
|
karafka_config.client_id = config.client_id
|
|
12
|
-
karafka_config.consumer_mapper = config.consumer_mapper_class.classify.constantize.new
|
|
13
11
|
karafka_config.kafka = config.to_kafka_options
|
|
14
12
|
|
|
15
13
|
karafka_config.pause_timeout = config.pause_timeout * 1_000 if config.pause_timeout.present?
|
|
@@ -43,13 +41,16 @@ class Sbmt::KafkaConsumer::ClientConfigurer
|
|
|
43
41
|
|
|
44
42
|
raise "No configured consumer groups found, exiting" if target_consumer_groups.blank?
|
|
45
43
|
|
|
44
|
+
consumer_mapper = config.consumer_mapper_class.classify.constantize.new
|
|
45
|
+
|
|
46
46
|
# clear routes in case CLI runner tries to reconfigure them
|
|
47
47
|
# but railtie initializer had already executed and did the same
|
|
48
48
|
# otherwise we'll get duplicate routes error from sbmt-karafka internal config validation process
|
|
49
49
|
Karafka::App.routes.clear
|
|
50
50
|
Karafka::App.routes.draw do
|
|
51
51
|
target_consumer_groups.each do |cg|
|
|
52
|
-
|
|
52
|
+
group_id = consumer_mapper.call(cg.name)
|
|
53
|
+
consumer_group group_id do
|
|
53
54
|
cg.topics.each do |t|
|
|
54
55
|
topic t.name do
|
|
55
56
|
active t.active
|
|
@@ -66,7 +67,7 @@ class Sbmt::KafkaConsumer::ClientConfigurer
|
|
|
66
67
|
|
|
67
68
|
def self.routes
|
|
68
69
|
Karafka::App.routes.map do |cg|
|
|
69
|
-
topics = cg.topics.map { |t| {name: t.name, deserializer: t.
|
|
70
|
+
topics = cg.topics.map { |t| {name: t.name, deserializer: t.deserializers.payload} }
|
|
70
71
|
{group: cg.id, topics: topics}
|
|
71
72
|
end
|
|
72
73
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sbmt
|
|
4
|
+
module KafkaConsumer
|
|
5
|
+
module Routing
|
|
6
|
+
module ConsumerMapper
|
|
7
|
+
class Base
|
|
8
|
+
# @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
|
|
9
|
+
# @return [String] remapped final consumer group name
|
|
10
|
+
def call(raw_consumer_group_name)
|
|
11
|
+
raise "Implement #call in a subclass"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "consumer_mapper/base"
|
|
4
|
+
|
|
1
5
|
module Sbmt
|
|
2
6
|
module KafkaConsumer
|
|
3
7
|
module Routing
|
|
4
|
-
class KarafkaV1ConsumerMapper <
|
|
8
|
+
class KarafkaV1ConsumerMapper < ConsumerMapper::Base
|
|
9
|
+
# karafka v1 consumer group name mapper
|
|
5
10
|
def call(raw_consumer_group_name)
|
|
6
11
|
client_id = ActiveSupport::Inflector.underscore(Karafka::App.config.client_id).tr("/", "_")
|
|
7
12
|
"#{client_id}_#{raw_consumer_group_name}"
|
|
@@ -1,9 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "consumer_mapper/base"
|
|
4
|
+
|
|
1
5
|
module Sbmt
|
|
2
6
|
module KafkaConsumer
|
|
3
7
|
module Routing
|
|
4
|
-
#
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
# karafka v2 (before 2.4) consumer group name mapper
|
|
9
|
+
class KarafkaV2ConsumerMapper < ConsumerMapper::Base
|
|
10
|
+
def call(raw_consumer_group_name)
|
|
11
|
+
"#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
7
14
|
end
|
|
8
15
|
end
|
|
9
16
|
end
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
RSpec.shared_context "with sbmt karafka consumer" do
|
|
4
4
|
subject(:consume_with_sbmt_karafka) do
|
|
5
|
-
coordinator.increment
|
|
5
|
+
coordinator.increment(:consume)
|
|
6
6
|
consumer.on_consume
|
|
7
7
|
end
|
|
8
8
|
|
|
@@ -27,27 +27,27 @@ RSpec.shared_context "with sbmt karafka consumer" do
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
def publish_to_sbmt_karafka(raw_payload, opts = {})
|
|
30
|
-
message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(
|
|
30
|
+
message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(build_metadata_hash(opts)))
|
|
31
31
|
consumer.messages = consumer_messages([message])
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
def publish_to_sbmt_karafka_batch(raw_payloads, opts = {})
|
|
35
35
|
messages = raw_payloads.map do |p|
|
|
36
|
-
Karafka::Messages::Message.new(p, Karafka::Messages::Metadata.new(
|
|
36
|
+
Karafka::Messages::Message.new(p, Karafka::Messages::Metadata.new(build_metadata_hash(opts)))
|
|
37
37
|
end
|
|
38
38
|
consumer.messages = consumer_messages(messages)
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
# @return [Hash] message default options
|
|
42
|
-
def
|
|
42
|
+
def build_metadata_hash(opts)
|
|
43
43
|
{
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
offset: 0,
|
|
48
|
-
partition: 0,
|
|
49
|
-
received_at: Time.current,
|
|
50
|
-
topic: test_topic.name
|
|
44
|
+
deserializers: test_topic.deserializers(payload: opts[:deserializer] || null_deserializer),
|
|
45
|
+
raw_headers: opts[:headers] || {},
|
|
46
|
+
raw_key: opts[:key],
|
|
47
|
+
offset: opts[:offset] || 0,
|
|
48
|
+
partition: opts[:partition] || 0,
|
|
49
|
+
received_at: opts[:received_at] || Time.current,
|
|
50
|
+
topic: opts[:topic] || test_topic.name
|
|
51
51
|
}
|
|
52
52
|
end
|
|
53
53
|
|
data/sbmt-kafka_consumer.gemspec
CHANGED
|
@@ -6,12 +6,12 @@ Gem::Specification.new do |spec|
|
|
|
6
6
|
spec.name = "sbmt-kafka_consumer"
|
|
7
7
|
spec.license = "MIT"
|
|
8
8
|
spec.version = Sbmt::KafkaConsumer::VERSION
|
|
9
|
-
spec.authors = ["
|
|
9
|
+
spec.authors = ["Kuper Ruby-Platform Team"]
|
|
10
10
|
|
|
11
11
|
spec.summary = "Ruby gem for consuming Kafka messages"
|
|
12
12
|
spec.description = "This gem is used for consuming Kafka messages. It represents a wrapper over Karafka gem and is recommended for using as a transport with sbmt-outbox"
|
|
13
|
-
spec.homepage = "https://github.com/
|
|
14
|
-
spec.required_ruby_version = ">=
|
|
13
|
+
spec.homepage = "https://github.com/Kuper-Tech/sbmt-kafka_consumer"
|
|
14
|
+
spec.required_ruby_version = ">= 3.0.0"
|
|
15
15
|
|
|
16
16
|
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
|
17
17
|
|
|
@@ -31,9 +31,9 @@ Gem::Specification.new do |spec|
|
|
|
31
31
|
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
32
32
|
spec.require_paths = ["lib"]
|
|
33
33
|
|
|
34
|
-
spec.add_dependency "rails", ">= 6.
|
|
34
|
+
spec.add_dependency "rails", ">= 6.1"
|
|
35
35
|
spec.add_dependency "zeitwerk", "~> 2.3"
|
|
36
|
-
spec.add_dependency "karafka", "~> 2.
|
|
36
|
+
spec.add_dependency "karafka", "~> 2.4"
|
|
37
37
|
spec.add_dependency "yabeda", ">= 0.11"
|
|
38
38
|
spec.add_dependency "anyway_config", ">= 2.4.0"
|
|
39
39
|
spec.add_dependency "thor"
|
metadata
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sbmt-kafka_consumer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- Kuper Ruby-Platform Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
@@ -16,14 +16,14 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '6.
|
|
19
|
+
version: '6.1'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '6.
|
|
26
|
+
version: '6.1'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: zeitwerk
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -43,9 +43,6 @@ dependencies:
|
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - "~>"
|
|
46
|
-
- !ruby/object:Gem::Version
|
|
47
|
-
version: '2.2'
|
|
48
|
-
- - "<"
|
|
49
46
|
- !ruby/object:Gem::Version
|
|
50
47
|
version: '2.4'
|
|
51
48
|
type: :runtime
|
|
@@ -53,9 +50,6 @@ dependencies:
|
|
|
53
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
54
51
|
requirements:
|
|
55
52
|
- - "~>"
|
|
56
|
-
- !ruby/object:Gem::Version
|
|
57
|
-
version: '2.2'
|
|
58
|
-
- - "<"
|
|
59
53
|
- !ruby/object:Gem::Version
|
|
60
54
|
version: '2.4'
|
|
61
55
|
- !ruby/object:Gem::Dependency
|
|
@@ -531,6 +525,7 @@ files:
|
|
|
531
525
|
- lib/sbmt/kafka_consumer/probes/host.rb
|
|
532
526
|
- lib/sbmt/kafka_consumer/probes/probe.rb
|
|
533
527
|
- lib/sbmt/kafka_consumer/railtie.rb
|
|
528
|
+
- lib/sbmt/kafka_consumer/routing/consumer_mapper/base.rb
|
|
534
529
|
- lib/sbmt/kafka_consumer/routing/karafka_v1_consumer_mapper.rb
|
|
535
530
|
- lib/sbmt/kafka_consumer/routing/karafka_v2_consumer_mapper.rb
|
|
536
531
|
- lib/sbmt/kafka_consumer/serialization/base_deserializer.rb
|
|
@@ -546,14 +541,14 @@ files:
|
|
|
546
541
|
- lib/sbmt/kafka_consumer/yabeda_configurer.rb
|
|
547
542
|
- rubocop/rspec.yml
|
|
548
543
|
- sbmt-kafka_consumer.gemspec
|
|
549
|
-
homepage: https://github.com/
|
|
544
|
+
homepage: https://github.com/Kuper-Tech/sbmt-kafka_consumer
|
|
550
545
|
licenses:
|
|
551
546
|
- MIT
|
|
552
547
|
metadata:
|
|
553
548
|
allowed_push_host: https://rubygems.org
|
|
554
|
-
homepage_uri: https://github.com/
|
|
555
|
-
source_code_uri: https://github.com/
|
|
556
|
-
changelog_uri: https://github.com/
|
|
549
|
+
homepage_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer
|
|
550
|
+
source_code_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer
|
|
551
|
+
changelog_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer/blob/master/CHANGELOG.md
|
|
557
552
|
rubygems_mfa_required: 'false'
|
|
558
553
|
post_install_message:
|
|
559
554
|
rdoc_options: []
|
|
@@ -563,7 +558,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
563
558
|
requirements:
|
|
564
559
|
- - ">="
|
|
565
560
|
- !ruby/object:Gem::Version
|
|
566
|
-
version:
|
|
561
|
+
version: 3.0.0
|
|
567
562
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
568
563
|
requirements:
|
|
569
564
|
- - ">="
|