karafka-testing 2.6.1 → 2.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +10 -11
- data/.github/workflows/push.yml +2 -2
- data/.rubocop.yml +1 -1
- data/.ruby-version +1 -1
- data/CHANGELOG.md +11 -0
- data/Gemfile +0 -1
- data/Gemfile.lock +11 -13
- data/README.md +4 -4
- data/karafka-testing.gemspec +1 -1
- data/lib/karafka/testing/helpers.rb +17 -3
- data/lib/karafka/testing/minitest/helpers.rb +65 -21
- data/lib/karafka/testing/minitest/proxy.rb +3 -0
- data/lib/karafka/testing/rspec/helpers.rb +40 -24
- data/lib/karafka/testing/rspec/proxy.rb +3 -0
- data/lib/karafka/testing/spec_producer_client.rb +5 -5
- data/lib/karafka/testing/version.rb +1 -1
- data/lib/karafka/testing.rb +6 -0
- data/package-lock.json +9 -9
- data/renovate.json +3 -0
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2be14092c2b929e48d6849d330f913b9000006217a48a2ac0d9cbadc1a43fe77
|
|
4
|
+
data.tar.gz: c575f2aee39ad9c80ac92c32729572cc882d6951ec5adbb056fc690ca330cd1f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 384fc90788d47afbfa37d7fddb81a229c89fdfc5157b433f0cf5124bbba0309d9cb099c6fb63d3678eba786334352cad8cd257b09227f37b084baf54d88dd7ad
|
|
7
|
+
data.tar.gz: 5a11804abcad1b3987e5d4c77a77c12cc9d39568cf3c833dcc0286300af64e077dd728bfda5b2cba714445623d9ee93b9ab4d22ba2d84ddb8b10c1a84e05f48c
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -24,7 +24,6 @@ jobs:
|
|
|
24
24
|
- '4.0'
|
|
25
25
|
- '3.4'
|
|
26
26
|
- '3.3'
|
|
27
|
-
- '3.2'
|
|
28
27
|
include:
|
|
29
28
|
- ruby: '4.0'
|
|
30
29
|
coverage: 'true'
|
|
@@ -37,7 +36,7 @@ jobs:
|
|
|
37
36
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
38
37
|
|
|
39
38
|
- name: Set up Ruby
|
|
40
|
-
uses: ruby/setup-ruby@
|
|
39
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
41
40
|
with:
|
|
42
41
|
ruby-version: ${{matrix.ruby}}
|
|
43
42
|
bundler: 'latest'
|
|
@@ -67,9 +66,9 @@ jobs:
|
|
|
67
66
|
with:
|
|
68
67
|
fetch-depth: 0
|
|
69
68
|
- name: Set up Ruby
|
|
70
|
-
uses: ruby/setup-ruby@
|
|
69
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
71
70
|
with:
|
|
72
|
-
ruby-version: '4.0.
|
|
71
|
+
ruby-version: '4.0.7'
|
|
73
72
|
bundler-cache: true
|
|
74
73
|
- name: Run rubocop
|
|
75
74
|
run: bundle exec rubocop
|
|
@@ -82,9 +81,9 @@ jobs:
|
|
|
82
81
|
steps:
|
|
83
82
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
84
83
|
- name: Set up Ruby
|
|
85
|
-
uses: ruby/setup-ruby@
|
|
84
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
86
85
|
with:
|
|
87
|
-
ruby-version: '4.0.
|
|
86
|
+
ruby-version: '4.0.7'
|
|
88
87
|
bundler-cache: true
|
|
89
88
|
- run: bundle exec yard-lint lib/
|
|
90
89
|
|
|
@@ -96,9 +95,9 @@ jobs:
|
|
|
96
95
|
with:
|
|
97
96
|
fetch-depth: 0
|
|
98
97
|
- name: Set up Ruby
|
|
99
|
-
uses: ruby/setup-ruby@
|
|
98
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
100
99
|
with:
|
|
101
|
-
ruby-version: '4.0.
|
|
100
|
+
ruby-version: '4.0.7'
|
|
102
101
|
bundler: 'latest'
|
|
103
102
|
- name: Install dependencies
|
|
104
103
|
run: bundle install --jobs 4 --retry 3
|
|
@@ -113,9 +112,9 @@ jobs:
|
|
|
113
112
|
with:
|
|
114
113
|
fetch-depth: 0
|
|
115
114
|
- name: Set up Ruby
|
|
116
|
-
uses: ruby/setup-ruby@
|
|
115
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
117
116
|
with:
|
|
118
|
-
ruby-version: '4.0.
|
|
117
|
+
ruby-version: '4.0.7'
|
|
119
118
|
bundler: 'latest'
|
|
120
119
|
- name: Install dependencies
|
|
121
120
|
run: bundle install --jobs 4 --retry 3
|
|
@@ -130,7 +129,7 @@ jobs:
|
|
|
130
129
|
with:
|
|
131
130
|
fetch-depth: 0
|
|
132
131
|
- name: Set up Node.js
|
|
133
|
-
uses: actions/setup-node@
|
|
132
|
+
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
|
134
133
|
with:
|
|
135
134
|
node-version: '20'
|
|
136
135
|
cache: 'npm'
|
data/.github/workflows/push.yml
CHANGED
|
@@ -24,7 +24,7 @@ jobs:
|
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1.331.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -32,4 +32,4 @@ jobs:
|
|
|
32
32
|
run: |
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
|
-
- uses: rubygems/release-gem@
|
|
35
|
+
- uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
|
data/.rubocop.yml
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.7
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Karafka Testing Changelog
|
|
2
2
|
|
|
3
|
+
## 2.6.3 (2026-10-25)
|
|
4
|
+
- **[EOL]** Drop Ruby 3.2 support; require Ruby 3.3+.
|
|
5
|
+
- [Maintenance] Align internal naming and YARD docs with the Karafka ecosystem conventions.
|
|
6
|
+
- [Maintenance] Drop `ostruct` from the test dependencies.
|
|
7
|
+
- [Fix] Make the Minitest helpers work without Mocha. They no longer fail with `undefined method 'stubs'` when Mocha is not loaded, and still use Mocha when it is.
|
|
8
|
+
- [Fix] Set `metadata.message` on test messages and give `headers: nil` empty `raw_headers`, as a real run does.
|
|
9
|
+
- [Fix] Make `produce(partition: N)` set N as the batch and consumer partition, not only the message one. With mixed partitions the first message decides.
|
|
10
|
+
|
|
11
|
+
## 2.6.2 (2026-08-13)
|
|
12
|
+
- [Fix] Read the `strategy_selector` and `expansions_selector` settings from their new `consumer_groups` location. Requires Karafka `>= 2.6.0`.
|
|
13
|
+
|
|
3
14
|
## 2.6.1 (2026-08-05)
|
|
4
15
|
- [Fix] Fix the dependency constraints.
|
|
5
16
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
karafka-testing (2.6.
|
|
4
|
+
karafka-testing (2.6.3)
|
|
5
5
|
karafka (>= 2.6.0.rc2, < 2.7.0)
|
|
6
6
|
waterdrop (>= 2.10.0)
|
|
7
7
|
|
|
@@ -12,22 +12,22 @@ GEM
|
|
|
12
12
|
drb (2.2.3)
|
|
13
13
|
ffi (1.17.4)
|
|
14
14
|
ffi (1.17.4-x86_64-linux-gnu)
|
|
15
|
-
json (
|
|
16
|
-
karafka (2.6.
|
|
17
|
-
karafka-core (>= 2.6.
|
|
15
|
+
json (3.0.2)
|
|
16
|
+
karafka (2.6.1)
|
|
17
|
+
karafka-core (>= 2.6.2, < 2.7.0)
|
|
18
18
|
karafka-rdkafka (>= 0.28.0)
|
|
19
|
-
waterdrop (>= 2.10.
|
|
19
|
+
waterdrop (>= 2.10.2, < 3.0.0)
|
|
20
20
|
zeitwerk (~> 2.3)
|
|
21
|
-
karafka-core (2.6.
|
|
21
|
+
karafka-core (2.6.4)
|
|
22
22
|
karafka-rdkafka (>= 0.20.0)
|
|
23
23
|
logger (>= 1.6.0)
|
|
24
|
-
karafka-rdkafka (0.
|
|
24
|
+
karafka-rdkafka (0.30.1)
|
|
25
25
|
ffi (~> 1.17.1)
|
|
26
26
|
json (> 2.0)
|
|
27
27
|
logger
|
|
28
28
|
mini_portile2 (~> 2.6)
|
|
29
29
|
rake (> 12)
|
|
30
|
-
karafka-rdkafka (0.
|
|
30
|
+
karafka-rdkafka (0.30.1-x86_64-linux-gnu)
|
|
31
31
|
ffi (~> 1.17.1)
|
|
32
32
|
json (> 2.0)
|
|
33
33
|
logger
|
|
@@ -40,7 +40,6 @@ GEM
|
|
|
40
40
|
prism (~> 1.5)
|
|
41
41
|
mocha (3.1.0)
|
|
42
42
|
ruby2_keywords (>= 0.0.5)
|
|
43
|
-
ostruct (0.6.3)
|
|
44
43
|
prism (1.9.0)
|
|
45
44
|
rake (13.4.2)
|
|
46
45
|
rspec (3.13.2)
|
|
@@ -57,13 +56,13 @@ GEM
|
|
|
57
56
|
rspec-support (~> 3.13.0)
|
|
58
57
|
rspec-support (3.13.7)
|
|
59
58
|
ruby2_keywords (0.0.5)
|
|
60
|
-
simplecov (1.
|
|
59
|
+
simplecov (1.3.2)
|
|
61
60
|
warning (1.6.0)
|
|
62
|
-
waterdrop (2.10.
|
|
61
|
+
waterdrop (2.10.5)
|
|
63
62
|
karafka-core (>= 2.5.12, < 3.0.0)
|
|
64
63
|
karafka-rdkafka (>= 0.24.0)
|
|
65
64
|
zeitwerk (~> 2.3)
|
|
66
|
-
zeitwerk (2.8.
|
|
65
|
+
zeitwerk (2.8.3)
|
|
67
66
|
|
|
68
67
|
PLATFORMS
|
|
69
68
|
ruby
|
|
@@ -73,7 +72,6 @@ DEPENDENCIES
|
|
|
73
72
|
karafka-testing!
|
|
74
73
|
minitest
|
|
75
74
|
mocha
|
|
76
|
-
ostruct
|
|
77
75
|
rspec
|
|
78
76
|
simplecov
|
|
79
77
|
warning
|
data/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Karafka Testing library
|
|
2
2
|
|
|
3
|
-
[](https://github.com/karafka/karafka-testing/actions
|
|
4
|
-
[](
|
|
3
|
+
[](https://github.com/karafka/karafka-testing/actions/workflows/ci.yml)
|
|
4
|
+
[](https://badge.fury.io/rb/karafka-testing)
|
|
5
5
|
[](https://slack.karafka.io)
|
|
6
6
|
|
|
7
|
-
Karafka-Testing is a library that provides RSpec helpers
|
|
7
|
+
Karafka-Testing is a library that provides RSpec and Minitest helpers that make testing of Karafka consumers and producers much easier.
|
|
8
8
|
|
|
9
9
|
## Installation and usage
|
|
10
10
|
|
|
@@ -14,6 +14,6 @@ Karafka-Testing is a library that provides RSpec helpers, to make testing of Kar
|
|
|
14
14
|
|
|
15
15
|
First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
|
|
16
16
|
|
|
17
|
-
Each pull request must pass all the
|
|
17
|
+
Each pull request must pass all the tests, including integration tests, and meet our quality requirements.
|
|
18
18
|
|
|
19
19
|
Fork it, update and wait for the Github Actions results.
|
data/karafka-testing.gemspec
CHANGED
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
|
|
|
22
22
|
spec.add_dependency "karafka", ">= 2.6.0.rc2", "< 2.7.0"
|
|
23
23
|
spec.add_dependency "waterdrop", ">= 2.10.0"
|
|
24
24
|
|
|
25
|
-
spec.required_ruby_version = ">= 3.
|
|
25
|
+
spec.required_ruby_version = ">= 3.3.0"
|
|
26
26
|
|
|
27
27
|
spec.metadata = {
|
|
28
28
|
"homepage_uri" => "https://karafka.io",
|
|
@@ -8,19 +8,19 @@ module Karafka
|
|
|
8
8
|
# Finds all the routing topics matching requested topic within all topics or within
|
|
9
9
|
# provided consumer group based on name
|
|
10
10
|
#
|
|
11
|
-
# @param
|
|
11
|
+
# @param topic_name [String] requested topic name
|
|
12
12
|
# @param requested_consumer_group [String] requested consumer group or nil to look in all
|
|
13
13
|
# @return [Array<Karafka::Routing::Topic>] all matching topics
|
|
14
14
|
#
|
|
15
15
|
# @note Since we run the lookup on subscription groups, the search will automatically
|
|
16
16
|
# expand with matching patterns
|
|
17
|
-
def karafka_consumer_find_candidate_topics(
|
|
17
|
+
def karafka_consumer_find_candidate_topics(topic_name, requested_consumer_group)
|
|
18
18
|
subscription_groups = karafka_consumer_find_subscription_groups(requested_consumer_group)
|
|
19
19
|
|
|
20
20
|
candidate_topics = []
|
|
21
21
|
|
|
22
22
|
subscription_groups.each do |group|
|
|
23
|
-
topic = group.topics.find(
|
|
23
|
+
topic = group.topics.find(topic_name.to_s)
|
|
24
24
|
|
|
25
25
|
next unless topic
|
|
26
26
|
|
|
@@ -36,6 +36,20 @@ module Karafka
|
|
|
36
36
|
candidate_topics.uniq { |topic| topic.subscription_group.name }
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
+
# Builds a coordinator for given topic partition, as a consumer gets in a real run
|
|
40
|
+
#
|
|
41
|
+
# @param topic [Karafka::Routing::Topic] topic of the consumer
|
|
42
|
+
# @param partition [Integer] partition the consumer works on
|
|
43
|
+
# @return [Karafka::Processing::ConsumerGroups::Coordinator] coordinator for the partition
|
|
44
|
+
def karafka_build_coordinator(topic, partition)
|
|
45
|
+
coordinator = Karafka::Processing::CoordinatorsBuffer
|
|
46
|
+
.new(Karafka::Routing::Topics.new([topic]))
|
|
47
|
+
.find_or_create(topic.name, partition)
|
|
48
|
+
|
|
49
|
+
coordinator.seek_offset = 0
|
|
50
|
+
coordinator
|
|
51
|
+
end
|
|
52
|
+
|
|
39
53
|
# Finds subscription groups from the requested consumer group or selects all if no
|
|
40
54
|
# consumer group specified
|
|
41
55
|
# @param requested_consumer_group [String] requested consumer group or nil to look in all
|
|
@@ -39,7 +39,23 @@ module Karafka
|
|
|
39
39
|
|
|
40
40
|
@_karafka_consumer_mappings = {}
|
|
41
41
|
|
|
42
|
-
Karafka
|
|
42
|
+
# Mocha is a dev dependency of many apps without being used in their Karafka tests,
|
|
43
|
+
# so we stub with it only when its patches are actually available
|
|
44
|
+
if Object.const_defined?("Mocha", false) && Karafka.producer.respond_to?(:stubs)
|
|
45
|
+
Karafka.producer.stubs(:client).returns(@_karafka_producer_client)
|
|
46
|
+
else
|
|
47
|
+
producer_client = @_karafka_producer_client
|
|
48
|
+
Karafka.producer.define_singleton_method(:client) { producer_client }
|
|
49
|
+
@_karafka_producer_client_stubbed = true
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Mocha removes its own stubs, so we only remove the one we defined
|
|
54
|
+
teardown_flow = lambda do
|
|
55
|
+
next unless @_karafka_producer_client_stubbed
|
|
56
|
+
|
|
57
|
+
Karafka.producer.singleton_class.remove_method(:client)
|
|
58
|
+
@_karafka_producer_client_stubbed = false
|
|
43
59
|
end
|
|
44
60
|
|
|
45
61
|
if base.to_s == "Minitest::Spec"
|
|
@@ -60,29 +76,44 @@ module Karafka
|
|
|
60
76
|
|
|
61
77
|
base.prepend(setup_mod)
|
|
62
78
|
end
|
|
79
|
+
|
|
80
|
+
# `after_teardown` runs even when a test overrides `teardown` without calling `super`
|
|
81
|
+
teardown_mod = Module.new do
|
|
82
|
+
define_method(:after_teardown) do
|
|
83
|
+
super()
|
|
84
|
+
ensure
|
|
85
|
+
instance_exec(&teardown_flow)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
base.prepend(teardown_mod)
|
|
63
90
|
end
|
|
64
91
|
end
|
|
65
92
|
|
|
66
93
|
# Creates a consumer instance for a given topic
|
|
67
94
|
#
|
|
68
|
-
# @param
|
|
69
|
-
#
|
|
95
|
+
# @param topic_name [String, Symbol] name of the topic for which we want to create a
|
|
96
|
+
# consumer instance
|
|
70
97
|
# @param requested_consumer_group [String, Symbol, nil] optional name of the consumer group
|
|
71
98
|
# if we have multiple consumer groups listening on the same topic
|
|
72
99
|
# @return [Object] Karafka consumer instance
|
|
73
100
|
# @raise [Karafka::Testing::Errors::TopicNotFoundError] raised when we're unable to find
|
|
74
101
|
# topic that was requested
|
|
102
|
+
# @raise [Karafka::Testing::Errors::TopicInManyConsumerGroupsError] raised when the topic is
|
|
103
|
+
# in many consumer groups and no consumer group was requested
|
|
104
|
+
# @raise [Karafka::Testing::Errors::ConsumerGroupNotFoundError] raised when the requested
|
|
105
|
+
# consumer group does not exist
|
|
75
106
|
#
|
|
76
107
|
# @example Creates a consumer instance with settings for `my_requested_topic`
|
|
77
108
|
# consumer = @karafka.consumer_for(:my_requested_topic)
|
|
78
|
-
def _karafka_consumer_for(
|
|
109
|
+
def _karafka_consumer_for(topic_name, requested_consumer_group = nil)
|
|
79
110
|
selected_topics = Testing::Helpers.karafka_consumer_find_candidate_topics(
|
|
80
|
-
|
|
111
|
+
topic_name.to_s,
|
|
81
112
|
requested_consumer_group.to_s
|
|
82
113
|
)
|
|
83
114
|
|
|
84
|
-
raise Errors::TopicInManyConsumerGroupsError,
|
|
85
|
-
raise Errors::TopicNotFoundError,
|
|
115
|
+
raise Errors::TopicInManyConsumerGroupsError, topic_name if selected_topics.size > 1
|
|
116
|
+
raise Errors::TopicNotFoundError, topic_name if selected_topics.empty?
|
|
86
117
|
|
|
87
118
|
_karafka_build_consumer_for(selected_topics.first)
|
|
88
119
|
end
|
|
@@ -96,7 +127,7 @@ module Karafka
|
|
|
96
127
|
# @karafka.produce({ 'hello' => 'world' }.to_json)
|
|
97
128
|
#
|
|
98
129
|
# @example Send a json message to consumer and simulate, that it is partition 6
|
|
99
|
-
# @karafka.produce({ 'hello' => 'world' }.to_json,
|
|
130
|
+
# @karafka.produce({ 'hello' => 'world' }.to_json, partition: 6)
|
|
100
131
|
def _karafka_add_message_to_consumer_if_needed(message)
|
|
101
132
|
consumer_obj = if message[:consumer_group]
|
|
102
133
|
_karafka_find_consumer_for_message(message)
|
|
@@ -129,16 +160,33 @@ module Karafka
|
|
|
129
160
|
metadata[key] = message.fetch(message_key)
|
|
130
161
|
end
|
|
131
162
|
|
|
132
|
-
|
|
133
|
-
|
|
163
|
+
metadata[:raw_headers] ||= {}
|
|
164
|
+
|
|
165
|
+
metadata = Karafka::Messages::Metadata.new(metadata)
|
|
166
|
+
karafka_message = Karafka::Messages::Message.new(
|
|
134
167
|
message[:payload],
|
|
135
|
-
|
|
168
|
+
metadata
|
|
136
169
|
)
|
|
170
|
+
metadata[:message] = karafka_message
|
|
171
|
+
|
|
172
|
+
# Add this message to previously produced messages
|
|
173
|
+
@_karafka_consumer_messages << karafka_message
|
|
174
|
+
# A real batch never mixes partitions, so the first message of this consumer decides
|
|
175
|
+
# the partition
|
|
176
|
+
partition = (@_karafka_batch_partitions ||= {})[consumer_obj] ||= karafka_message.partition
|
|
177
|
+
|
|
178
|
+
if consumer_obj.partition != partition
|
|
179
|
+
consumer_obj.coordinator = Testing::Helpers.karafka_build_coordinator(
|
|
180
|
+
consumer_obj.topic,
|
|
181
|
+
partition
|
|
182
|
+
)
|
|
183
|
+
end
|
|
184
|
+
|
|
137
185
|
# Update batch metadata
|
|
138
186
|
batch_metadata = Karafka::Messages::Builders::BatchMetadata.call(
|
|
139
187
|
@_karafka_consumer_messages,
|
|
140
188
|
consumer_obj.topic,
|
|
141
|
-
|
|
189
|
+
partition,
|
|
142
190
|
Time.now
|
|
143
191
|
)
|
|
144
192
|
|
|
@@ -159,7 +207,7 @@ module Karafka
|
|
|
159
207
|
# WaterDrop message attribute (Kafka assigns offsets, not producers)
|
|
160
208
|
@_karafka_next_offset = metadata.delete(:offset)
|
|
161
209
|
|
|
162
|
-
|
|
210
|
+
topic_name = if metadata[:topic]
|
|
163
211
|
metadata[:topic]
|
|
164
212
|
elsif defined?(@consumer)
|
|
165
213
|
@consumer.topic.name
|
|
@@ -169,7 +217,7 @@ module Karafka
|
|
|
169
217
|
end
|
|
170
218
|
Karafka.producer.produce_sync(
|
|
171
219
|
{
|
|
172
|
-
topic:
|
|
220
|
+
topic: topic_name,
|
|
173
221
|
payload: payload
|
|
174
222
|
}.merge(metadata)
|
|
175
223
|
)
|
|
@@ -254,21 +302,17 @@ module Karafka
|
|
|
254
302
|
# @param topic [Karafka::Routing::Topic] topic for which we want to build the consumer
|
|
255
303
|
# @return [Object] karafka consumer
|
|
256
304
|
def _karafka_build_consumer_for(topic)
|
|
257
|
-
coordinators = Karafka::Processing::CoordinatorsBuffer.new(
|
|
258
|
-
Karafka::Routing::Topics.new([topic])
|
|
259
|
-
)
|
|
260
305
|
processing_cfg = Karafka::App.config.internal.processing
|
|
261
306
|
@consumer = topic.consumer.new
|
|
262
307
|
@consumer.producer = Karafka::App.producer
|
|
263
308
|
# Inject appropriate strategy so needed options and components are available
|
|
264
|
-
strategy =
|
|
309
|
+
strategy = processing_cfg.consumer_groups.strategy_selector.find(topic)
|
|
265
310
|
@consumer.singleton_class.include(strategy)
|
|
266
311
|
@consumer.client = @_karafka_consumer_client
|
|
267
|
-
@consumer.coordinator =
|
|
268
|
-
@consumer.coordinator.seek_offset = 0
|
|
312
|
+
@consumer.coordinator = Testing::Helpers.karafka_build_coordinator(topic, 0)
|
|
269
313
|
# Indicate usage as for tests no direct enqueuing happens
|
|
270
314
|
@consumer.instance_variable_set(:@used, true)
|
|
271
|
-
expansions = processing_cfg.expansions_selector.find(topic)
|
|
315
|
+
expansions = processing_cfg.consumer_groups.expansions_selector.find(topic)
|
|
272
316
|
expansions.each { |expansion| @consumer.singleton_class.include(expansion) }
|
|
273
317
|
@_karafka_consumer_mappings[topic.id] = @consumer
|
|
274
318
|
@consumer
|
|
@@ -12,16 +12,19 @@ module Karafka
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
# Forwards all arguments to `#_karafka_consumer_for`
|
|
15
|
+
# @return [Karafka::BaseConsumer] consumer instance
|
|
15
16
|
def consumer_for(*)
|
|
16
17
|
@minitest_example._karafka_consumer_for(*)
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# Forwards all arguments to `#_karafka_produce`
|
|
21
|
+
# @return [Rdkafka::Producer::DeliveryReport] delivery report
|
|
20
22
|
def produce(*)
|
|
21
23
|
@minitest_example._karafka_produce(*)
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
# Forwards all arguments to `#_karafka_produce_to`
|
|
27
|
+
# @return [Rdkafka::Producer::DeliveryReport] delivery report
|
|
25
28
|
def produce_to(*)
|
|
26
29
|
@minitest_example._karafka_produce_to(*)
|
|
27
30
|
end
|
|
@@ -66,26 +66,30 @@ module Karafka
|
|
|
66
66
|
|
|
67
67
|
# Creates a consumer instance for a given topic
|
|
68
68
|
#
|
|
69
|
-
# @param
|
|
70
|
-
#
|
|
69
|
+
# @param topic_name [String, Symbol] name of the topic for which we want to create a
|
|
70
|
+
# consumer instance
|
|
71
71
|
# @param requested_consumer_group [String, Symbol, nil] optional name of the consumer group
|
|
72
72
|
# if we have multiple consumer groups listening on the same topic
|
|
73
73
|
# @return [Object] Karafka consumer instance
|
|
74
74
|
# @raise [Karafka::Testing::Errors::TopicNotFoundError] raised when we're unable to find
|
|
75
75
|
# topic that was requested
|
|
76
|
+
# @raise [Karafka::Testing::Errors::TopicInManyConsumerGroupsError] raised when the topic is
|
|
77
|
+
# in many consumer groups and no consumer group was requested
|
|
78
|
+
# @raise [Karafka::Testing::Errors::ConsumerGroupNotFoundError] raised when the requested
|
|
79
|
+
# consumer group does not exist
|
|
76
80
|
#
|
|
77
81
|
# @example Creates a MyConsumer consumer instance with settings for `my_requested_topic`
|
|
78
82
|
# RSpec.describe MyConsumer do
|
|
79
83
|
# subject(:consumer) { karafka.consumer_for(:my_requested_topic) }
|
|
80
84
|
# end
|
|
81
|
-
def _karafka_consumer_for(
|
|
85
|
+
def _karafka_consumer_for(topic_name, requested_consumer_group = nil)
|
|
82
86
|
selected_topics = Testing::Helpers.karafka_consumer_find_candidate_topics(
|
|
83
|
-
|
|
87
|
+
topic_name.to_s,
|
|
84
88
|
requested_consumer_group.to_s
|
|
85
89
|
)
|
|
86
90
|
|
|
87
|
-
raise Errors::TopicInManyConsumerGroupsError,
|
|
88
|
-
raise Errors::TopicNotFoundError,
|
|
91
|
+
raise Errors::TopicInManyConsumerGroupsError, topic_name if selected_topics.size > 1
|
|
92
|
+
raise Errors::TopicNotFoundError, topic_name if selected_topics.empty?
|
|
89
93
|
|
|
90
94
|
_karafka_build_consumer_for(selected_topics.first)
|
|
91
95
|
end
|
|
@@ -101,12 +105,12 @@ module Karafka
|
|
|
101
105
|
#
|
|
102
106
|
# @example Send a json message to consumer and simulate, that it is partition 6
|
|
103
107
|
# before do
|
|
104
|
-
# karafka.produce({ 'hello' => 'world' }.to_json,
|
|
108
|
+
# karafka.produce({ 'hello' => 'world' }.to_json, partition: 6)
|
|
105
109
|
# end
|
|
106
110
|
def _karafka_add_message_to_consumer_if_needed(message)
|
|
107
111
|
consumer_obj = if defined?(consumer)
|
|
108
112
|
consumer
|
|
109
|
-
elsif
|
|
113
|
+
elsif _karafka_described_class_consumer?
|
|
110
114
|
_karafka_resolve_subject_consumer || _karafka_find_consumer_for_message(message)
|
|
111
115
|
else
|
|
112
116
|
_karafka_find_consumer_for_message(message)
|
|
@@ -135,17 +139,34 @@ module Karafka
|
|
|
135
139
|
metadata[key] = message.fetch(message_key)
|
|
136
140
|
end
|
|
137
141
|
|
|
138
|
-
|
|
139
|
-
|
|
142
|
+
metadata[:raw_headers] ||= {}
|
|
143
|
+
|
|
144
|
+
metadata = Karafka::Messages::Metadata.new(metadata)
|
|
145
|
+
karafka_message = Karafka::Messages::Message.new(
|
|
140
146
|
message[:payload],
|
|
141
|
-
|
|
147
|
+
metadata
|
|
142
148
|
)
|
|
149
|
+
metadata[:message] = karafka_message
|
|
150
|
+
|
|
151
|
+
# Add this message to previously produced messages
|
|
152
|
+
_karafka_consumer_messages << karafka_message
|
|
153
|
+
|
|
154
|
+
# A real batch never mixes partitions, so the first message of this consumer decides
|
|
155
|
+
# the partition
|
|
156
|
+
partition = (@_karafka_batch_partitions ||= {})[consumer_obj] ||= karafka_message.partition
|
|
157
|
+
|
|
158
|
+
if consumer_obj.partition != partition
|
|
159
|
+
consumer_obj.coordinator = Testing::Helpers.karafka_build_coordinator(
|
|
160
|
+
consumer_obj.topic,
|
|
161
|
+
partition
|
|
162
|
+
)
|
|
163
|
+
end
|
|
143
164
|
|
|
144
165
|
# Update batch metadata
|
|
145
166
|
batch_metadata = Karafka::Messages::Builders::BatchMetadata.call(
|
|
146
167
|
_karafka_consumer_messages,
|
|
147
168
|
consumer_obj.topic,
|
|
148
|
-
|
|
169
|
+
partition,
|
|
149
170
|
Time.now
|
|
150
171
|
)
|
|
151
172
|
|
|
@@ -166,11 +187,11 @@ module Karafka
|
|
|
166
187
|
# WaterDrop message attribute (Kafka assigns offsets, not producers)
|
|
167
188
|
@_karafka_next_offset = metadata.delete(:offset)
|
|
168
189
|
|
|
169
|
-
|
|
190
|
+
topic_name = if metadata[:topic]
|
|
170
191
|
metadata[:topic]
|
|
171
192
|
elsif defined?(consumer)
|
|
172
193
|
consumer.topic.name
|
|
173
|
-
elsif
|
|
194
|
+
elsif _karafka_described_class_consumer? && (sub = _karafka_resolve_subject_consumer)
|
|
174
195
|
sub.topic.name
|
|
175
196
|
else
|
|
176
197
|
last_consumer = @_karafka_consumer_mappings&.values&.last
|
|
@@ -178,7 +199,7 @@ module Karafka
|
|
|
178
199
|
end
|
|
179
200
|
Karafka.producer.produce_sync(
|
|
180
201
|
{
|
|
181
|
-
topic:
|
|
202
|
+
topic: topic_name,
|
|
182
203
|
payload: payload
|
|
183
204
|
}.merge(metadata)
|
|
184
205
|
)
|
|
@@ -222,7 +243,7 @@ module Karafka
|
|
|
222
243
|
# When true, the implicit RSpec subject can be used as a consumer fallback.
|
|
223
244
|
#
|
|
224
245
|
# @return [Boolean] true if described_class inherits from Karafka::BaseConsumer
|
|
225
|
-
def
|
|
246
|
+
def _karafka_described_class_consumer?
|
|
226
247
|
respond_to?(:described_class) &&
|
|
227
248
|
described_class.is_a?(Class) &&
|
|
228
249
|
described_class < Karafka::BaseConsumer
|
|
@@ -286,22 +307,17 @@ module Karafka
|
|
|
286
307
|
# @param topic [Karafka::Routing::Topic] topic for which we want to build the consumer
|
|
287
308
|
# @return [Object] karafka consumer
|
|
288
309
|
def _karafka_build_consumer_for(topic)
|
|
289
|
-
coordinators = Karafka::Processing::CoordinatorsBuffer.new(
|
|
290
|
-
Karafka::Routing::Topics.new([topic])
|
|
291
|
-
)
|
|
292
|
-
|
|
293
310
|
processing_cfg = Karafka::App.config.internal.processing
|
|
294
311
|
consumer = topic.consumer.new
|
|
295
312
|
consumer.producer = Karafka::App.producer
|
|
296
313
|
# Inject appropriate strategy so needed options and components are available
|
|
297
|
-
strategy = processing_cfg.strategy_selector.find(topic)
|
|
314
|
+
strategy = processing_cfg.consumer_groups.strategy_selector.find(topic)
|
|
298
315
|
consumer.singleton_class.include(strategy)
|
|
299
316
|
consumer.client = _karafka_consumer_client
|
|
300
|
-
consumer.coordinator =
|
|
301
|
-
consumer.coordinator.seek_offset = 0
|
|
317
|
+
consumer.coordinator = Testing::Helpers.karafka_build_coordinator(topic, 0)
|
|
302
318
|
# Indicate usage as for tests no direct enqueuing happens
|
|
303
319
|
consumer.instance_variable_set(:@used, true)
|
|
304
|
-
expansions = processing_cfg.expansions_selector.find(topic)
|
|
320
|
+
expansions = processing_cfg.consumer_groups.expansions_selector.find(topic)
|
|
305
321
|
expansions.each { |expansion| consumer.singleton_class.include(expansion) }
|
|
306
322
|
|
|
307
323
|
@_karafka_consumer_mappings[topic.id] = consumer
|
|
@@ -12,16 +12,19 @@ module Karafka
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
# Forwards all arguments to `#_karafka_consumer_for`
|
|
15
|
+
# @return [Karafka::BaseConsumer] consumer instance
|
|
15
16
|
def consumer_for(*)
|
|
16
17
|
@rspec_example._karafka_consumer_for(*)
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# Forwards all arguments to `#_karafka_produce`
|
|
21
|
+
# @return [Rdkafka::Producer::DeliveryReport] delivery report
|
|
20
22
|
def produce(*)
|
|
21
23
|
@rspec_example._karafka_produce(*)
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
# Forwards all arguments to `#_karafka_produce_to`
|
|
27
|
+
# @return [Rdkafka::Producer::DeliveryReport] delivery report
|
|
25
28
|
def produce_to(*)
|
|
26
29
|
@rspec_example._karafka_produce_to(*)
|
|
27
30
|
end
|
|
@@ -4,18 +4,18 @@ module Karafka
|
|
|
4
4
|
module Testing
|
|
5
5
|
# Spec producer client used to buffer messages that we send out in specs
|
|
6
6
|
class SpecProducerClient < ::WaterDrop::Clients::Buffered
|
|
7
|
-
# @param
|
|
8
|
-
# on it that are
|
|
9
|
-
def initialize(
|
|
7
|
+
# @param example [RSpec::Core::ExampleGroup, Minitest::Test] test example we need to hold to
|
|
8
|
+
# trigger actions on it that are test context aware
|
|
9
|
+
def initialize(example)
|
|
10
10
|
super(nil)
|
|
11
|
-
@
|
|
11
|
+
@example = example
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
# "Produces" message to Kafka. That is, it acknowledges it locally, adds it to the internal
|
|
15
15
|
# buffer and adds it (if needed) into the current consumer messages buffer
|
|
16
16
|
# @param message [Hash] `Karafka.producer.produce_sync` message hash
|
|
17
17
|
def produce(message)
|
|
18
|
-
@
|
|
18
|
+
@example._karafka_add_message_to_consumer_if_needed(message)
|
|
19
19
|
|
|
20
20
|
super
|
|
21
21
|
end
|
data/lib/karafka/testing.rb
CHANGED
|
@@ -8,6 +8,8 @@ module Karafka
|
|
|
8
8
|
#
|
|
9
9
|
# You should never use karafka-testing primitives when framework is not loaded because
|
|
10
10
|
# testing lib stubs certain pieces of Karafka that need to be initialized.
|
|
11
|
+
#
|
|
12
|
+
# @raise [Karafka::Testing::Errors::KarafkaNotLoadedError] raised when Karafka is not loaded
|
|
11
13
|
def ensure_karafka_loaded!
|
|
12
14
|
return if ::Karafka.const_defined?("App", false)
|
|
13
15
|
|
|
@@ -19,6 +21,10 @@ module Karafka
|
|
|
19
21
|
|
|
20
22
|
# If you do not initialize Karafka always within your specs, do not include/use this lib
|
|
21
23
|
# in places where Karafka would not be loaded.
|
|
24
|
+
#
|
|
25
|
+
# @raise [Karafka::Testing::Errors::KarafkaNotLoadedError] raised when Karafka is not loaded
|
|
26
|
+
# @raise [Karafka::Testing::Errors::KarafkaNotInitializedError] raised when Karafka is still
|
|
27
|
+
# initializing
|
|
22
28
|
def ensure_karafka_initialized!
|
|
23
29
|
ensure_karafka_loaded!
|
|
24
30
|
|
data/package-lock.json
CHANGED
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
}
|
|
104
104
|
},
|
|
105
105
|
"node_modules/fastq": {
|
|
106
|
-
"version": "1.20.
|
|
107
|
-
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.
|
|
108
|
-
"integrity": "sha512-
|
|
106
|
+
"version": "1.20.3",
|
|
107
|
+
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
|
|
108
|
+
"integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
|
|
109
109
|
"dev": true,
|
|
110
110
|
"license": "ISC",
|
|
111
111
|
"dependencies": {
|
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.9.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
|
|
291
|
+
"integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
|
@@ -312,9 +312,9 @@
|
|
|
312
312
|
}
|
|
313
313
|
},
|
|
314
314
|
"node_modules/yaml": {
|
|
315
|
-
"version": "2.9.
|
|
316
|
-
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.
|
|
317
|
-
"integrity": "sha512-
|
|
315
|
+
"version": "2.9.1",
|
|
316
|
+
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
|
|
317
|
+
"integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
|
|
318
318
|
"dev": true,
|
|
319
319
|
"license": "ISC",
|
|
320
320
|
"bin": {
|
data/renovate.json
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: karafka-testing
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.6.
|
|
4
|
+
version: 2.6.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -104,14 +104,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
104
104
|
requirements:
|
|
105
105
|
- - ">="
|
|
106
106
|
- !ruby/object:Gem::Version
|
|
107
|
-
version: 3.
|
|
107
|
+
version: 3.3.0
|
|
108
108
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
109
109
|
requirements:
|
|
110
110
|
- - ">="
|
|
111
111
|
- !ruby/object:Gem::Version
|
|
112
112
|
version: '0'
|
|
113
113
|
requirements: []
|
|
114
|
-
rubygems_version: 4.0.
|
|
114
|
+
rubygems_version: 4.0.20
|
|
115
115
|
specification_version: 4
|
|
116
116
|
summary: Library which provides helpers for easier Karafka consumers tests
|
|
117
117
|
test_files: []
|