nulogy_message_bus_consumer 0.5.0 → 1.0.0.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/nulogy_message_bus_consumer/message.rb +13 -11
- data/lib/nulogy_message_bus_consumer/version.rb +1 -1
- metadata +11 -159
- data/spec/dummy/Rakefile +0 -6
- data/spec/dummy/app/assets/config/manifest.js +0 -3
- data/spec/dummy/app/assets/stylesheets/application.css +0 -15
- data/spec/dummy/app/channels/application_cable/channel.rb +0 -4
- data/spec/dummy/app/channels/application_cable/connection.rb +0 -4
- data/spec/dummy/app/controllers/application_controller.rb +0 -2
- data/spec/dummy/app/helpers/application_helper.rb +0 -2
- data/spec/dummy/app/javascript/packs/application.js +0 -15
- data/spec/dummy/app/jobs/application_job.rb +0 -7
- data/spec/dummy/app/mailers/application_mailer.rb +0 -4
- data/spec/dummy/app/models/application_record.rb +0 -3
- data/spec/dummy/app/views/layouts/application.html.erb +0 -14
- data/spec/dummy/app/views/layouts/mailer.html.erb +0 -13
- data/spec/dummy/app/views/layouts/mailer.text.erb +0 -1
- data/spec/dummy/bin/rails +0 -4
- data/spec/dummy/bin/rake +0 -4
- data/spec/dummy/bin/setup +0 -33
- data/spec/dummy/config/application.rb +0 -29
- data/spec/dummy/config/boot.rb +0 -5
- data/spec/dummy/config/cable.yml +0 -10
- data/spec/dummy/config/credentials/message-bus-us-east-1.key +0 -1
- data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc +0 -1
- data/spec/dummy/config/database.yml +0 -27
- data/spec/dummy/config/environment.rb +0 -5
- data/spec/dummy/config/environments/development.rb +0 -62
- data/spec/dummy/config/environments/production.rb +0 -112
- data/spec/dummy/config/environments/test.rb +0 -49
- data/spec/dummy/config/initializers/application_controller_renderer.rb +0 -8
- data/spec/dummy/config/initializers/assets.rb +0 -12
- data/spec/dummy/config/initializers/backtrace_silencers.rb +0 -7
- data/spec/dummy/config/initializers/content_security_policy.rb +0 -28
- data/spec/dummy/config/initializers/cookies_serializer.rb +0 -5
- data/spec/dummy/config/initializers/filter_parameter_logging.rb +0 -4
- data/spec/dummy/config/initializers/inflections.rb +0 -16
- data/spec/dummy/config/initializers/message_bus_consumer.rb +0 -5
- data/spec/dummy/config/initializers/mime_types.rb +0 -4
- data/spec/dummy/config/initializers/wrap_parameters.rb +0 -14
- data/spec/dummy/config/locales/en.yml +0 -33
- data/spec/dummy/config/puma.rb +0 -36
- data/spec/dummy/config/routes.rb +0 -3
- data/spec/dummy/config/spring.rb +0 -6
- data/spec/dummy/config/storage.yml +0 -34
- data/spec/dummy/config.ru +0 -5
- data/spec/dummy/db/schema.rb +0 -21
- data/spec/dummy/log/development.log +0 -4
- data/spec/dummy/log/production.log +0 -18
- data/spec/dummy/log/test.log +0 -6083
- data/spec/dummy/public/404.html +0 -67
- data/spec/dummy/public/422.html +0 -67
- data/spec/dummy/public/500.html +0 -66
- data/spec/dummy/public/apple-touch-icon-precomposed.png +0 -0
- data/spec/dummy/public/apple-touch-icon.png +0 -0
- data/spec/dummy/public/favicon.ico +0 -0
- data/spec/dummy/tmp/development_secret.txt +0 -1
- data/spec/integration/nulogy_message_bus_consumer/auditor_spec.rb +0 -59
- data/spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb +0 -41
- data/spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +0 -131
- data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb +0 -54
- data/spec/integration/nulogy_message_bus_consumer/steps/supervise_consumer_lag_spec.rb +0 -54
- data/spec/integration/test_topic_spec.rb +0 -39
- data/spec/spec_helper.rb +0 -49
- data/spec/support/kafka.rb +0 -74
- data/spec/support/middleware_tap.rb +0 -12
- data/spec/support/test_topic.rb +0 -48
- data/spec/unit/nulogy_message_bus_consumer/config_spec.rb +0 -20
- data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb +0 -35
- data/spec/unit/nulogy_message_bus_consumer/message_spec.rb +0 -84
- data/spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb +0 -49
- data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +0 -58
- data/spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb +0 -56
- data/spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb +0 -70
- data/spec/unit/nulogy_message_bus_consumer/steps/monitor_replication_lag/calculator_spec.rb +0 -63
- data/spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb +0 -35
- data/spec/unit/nulogy_message_bus_consumer_spec.rb +0 -30
data/spec/dummy/public/404.html
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
<!DOCTYPE html>
|
2
|
-
<html>
|
3
|
-
<head>
|
4
|
-
<title>The page you were looking for doesn't exist (404)</title>
|
5
|
-
<meta name="viewport" content="width=device-width,initial-scale=1">
|
6
|
-
<style>
|
7
|
-
.rails-default-error-page {
|
8
|
-
background-color: #EFEFEF;
|
9
|
-
color: #2E2F30;
|
10
|
-
text-align: center;
|
11
|
-
font-family: arial, sans-serif;
|
12
|
-
margin: 0;
|
13
|
-
}
|
14
|
-
|
15
|
-
.rails-default-error-page div.dialog {
|
16
|
-
width: 95%;
|
17
|
-
max-width: 33em;
|
18
|
-
margin: 4em auto 0;
|
19
|
-
}
|
20
|
-
|
21
|
-
.rails-default-error-page div.dialog > div {
|
22
|
-
border: 1px solid #CCC;
|
23
|
-
border-right-color: #999;
|
24
|
-
border-left-color: #999;
|
25
|
-
border-bottom-color: #BBB;
|
26
|
-
border-top: #B00100 solid 4px;
|
27
|
-
border-top-left-radius: 9px;
|
28
|
-
border-top-right-radius: 9px;
|
29
|
-
background-color: white;
|
30
|
-
padding: 7px 12% 0;
|
31
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
32
|
-
}
|
33
|
-
|
34
|
-
.rails-default-error-page h1 {
|
35
|
-
font-size: 100%;
|
36
|
-
color: #730E15;
|
37
|
-
line-height: 1.5em;
|
38
|
-
}
|
39
|
-
|
40
|
-
.rails-default-error-page div.dialog > p {
|
41
|
-
margin: 0 0 1em;
|
42
|
-
padding: 1em;
|
43
|
-
background-color: #F7F7F7;
|
44
|
-
border: 1px solid #CCC;
|
45
|
-
border-right-color: #999;
|
46
|
-
border-left-color: #999;
|
47
|
-
border-bottom-color: #999;
|
48
|
-
border-bottom-left-radius: 4px;
|
49
|
-
border-bottom-right-radius: 4px;
|
50
|
-
border-top-color: #DADADA;
|
51
|
-
color: #666;
|
52
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
53
|
-
}
|
54
|
-
</style>
|
55
|
-
</head>
|
56
|
-
|
57
|
-
<body class="rails-default-error-page">
|
58
|
-
<!-- This file lives in public/404.html -->
|
59
|
-
<div class="dialog">
|
60
|
-
<div>
|
61
|
-
<h1>The page you were looking for doesn't exist.</h1>
|
62
|
-
<p>You may have mistyped the address or the page may have moved.</p>
|
63
|
-
</div>
|
64
|
-
<p>If you are the application owner check the logs for more information.</p>
|
65
|
-
</div>
|
66
|
-
</body>
|
67
|
-
</html>
|
data/spec/dummy/public/422.html
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
<!DOCTYPE html>
|
2
|
-
<html>
|
3
|
-
<head>
|
4
|
-
<title>The change you wanted was rejected (422)</title>
|
5
|
-
<meta name="viewport" content="width=device-width,initial-scale=1">
|
6
|
-
<style>
|
7
|
-
.rails-default-error-page {
|
8
|
-
background-color: #EFEFEF;
|
9
|
-
color: #2E2F30;
|
10
|
-
text-align: center;
|
11
|
-
font-family: arial, sans-serif;
|
12
|
-
margin: 0;
|
13
|
-
}
|
14
|
-
|
15
|
-
.rails-default-error-page div.dialog {
|
16
|
-
width: 95%;
|
17
|
-
max-width: 33em;
|
18
|
-
margin: 4em auto 0;
|
19
|
-
}
|
20
|
-
|
21
|
-
.rails-default-error-page div.dialog > div {
|
22
|
-
border: 1px solid #CCC;
|
23
|
-
border-right-color: #999;
|
24
|
-
border-left-color: #999;
|
25
|
-
border-bottom-color: #BBB;
|
26
|
-
border-top: #B00100 solid 4px;
|
27
|
-
border-top-left-radius: 9px;
|
28
|
-
border-top-right-radius: 9px;
|
29
|
-
background-color: white;
|
30
|
-
padding: 7px 12% 0;
|
31
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
32
|
-
}
|
33
|
-
|
34
|
-
.rails-default-error-page h1 {
|
35
|
-
font-size: 100%;
|
36
|
-
color: #730E15;
|
37
|
-
line-height: 1.5em;
|
38
|
-
}
|
39
|
-
|
40
|
-
.rails-default-error-page div.dialog > p {
|
41
|
-
margin: 0 0 1em;
|
42
|
-
padding: 1em;
|
43
|
-
background-color: #F7F7F7;
|
44
|
-
border: 1px solid #CCC;
|
45
|
-
border-right-color: #999;
|
46
|
-
border-left-color: #999;
|
47
|
-
border-bottom-color: #999;
|
48
|
-
border-bottom-left-radius: 4px;
|
49
|
-
border-bottom-right-radius: 4px;
|
50
|
-
border-top-color: #DADADA;
|
51
|
-
color: #666;
|
52
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
53
|
-
}
|
54
|
-
</style>
|
55
|
-
</head>
|
56
|
-
|
57
|
-
<body class="rails-default-error-page">
|
58
|
-
<!-- This file lives in public/422.html -->
|
59
|
-
<div class="dialog">
|
60
|
-
<div>
|
61
|
-
<h1>The change you wanted was rejected.</h1>
|
62
|
-
<p>Maybe you tried to change something you didn't have access to.</p>
|
63
|
-
</div>
|
64
|
-
<p>If you are the application owner check the logs for more information.</p>
|
65
|
-
</div>
|
66
|
-
</body>
|
67
|
-
</html>
|
data/spec/dummy/public/500.html
DELETED
@@ -1,66 +0,0 @@
|
|
1
|
-
<!DOCTYPE html>
|
2
|
-
<html>
|
3
|
-
<head>
|
4
|
-
<title>We're sorry, but something went wrong (500)</title>
|
5
|
-
<meta name="viewport" content="width=device-width,initial-scale=1">
|
6
|
-
<style>
|
7
|
-
.rails-default-error-page {
|
8
|
-
background-color: #EFEFEF;
|
9
|
-
color: #2E2F30;
|
10
|
-
text-align: center;
|
11
|
-
font-family: arial, sans-serif;
|
12
|
-
margin: 0;
|
13
|
-
}
|
14
|
-
|
15
|
-
.rails-default-error-page div.dialog {
|
16
|
-
width: 95%;
|
17
|
-
max-width: 33em;
|
18
|
-
margin: 4em auto 0;
|
19
|
-
}
|
20
|
-
|
21
|
-
.rails-default-error-page div.dialog > div {
|
22
|
-
border: 1px solid #CCC;
|
23
|
-
border-right-color: #999;
|
24
|
-
border-left-color: #999;
|
25
|
-
border-bottom-color: #BBB;
|
26
|
-
border-top: #B00100 solid 4px;
|
27
|
-
border-top-left-radius: 9px;
|
28
|
-
border-top-right-radius: 9px;
|
29
|
-
background-color: white;
|
30
|
-
padding: 7px 12% 0;
|
31
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
32
|
-
}
|
33
|
-
|
34
|
-
.rails-default-error-page h1 {
|
35
|
-
font-size: 100%;
|
36
|
-
color: #730E15;
|
37
|
-
line-height: 1.5em;
|
38
|
-
}
|
39
|
-
|
40
|
-
.rails-default-error-page div.dialog > p {
|
41
|
-
margin: 0 0 1em;
|
42
|
-
padding: 1em;
|
43
|
-
background-color: #F7F7F7;
|
44
|
-
border: 1px solid #CCC;
|
45
|
-
border-right-color: #999;
|
46
|
-
border-left-color: #999;
|
47
|
-
border-bottom-color: #999;
|
48
|
-
border-bottom-left-radius: 4px;
|
49
|
-
border-bottom-right-radius: 4px;
|
50
|
-
border-top-color: #DADADA;
|
51
|
-
color: #666;
|
52
|
-
box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
|
53
|
-
}
|
54
|
-
</style>
|
55
|
-
</head>
|
56
|
-
|
57
|
-
<body class="rails-default-error-page">
|
58
|
-
<!-- This file lives in public/500.html -->
|
59
|
-
<div class="dialog">
|
60
|
-
<div>
|
61
|
-
<h1>We're sorry, but something went wrong.</h1>
|
62
|
-
</div>
|
63
|
-
<p>If you are the application owner check the logs for more information.</p>
|
64
|
-
</div>
|
65
|
-
</body>
|
66
|
-
</html>
|
File without changes
|
File without changes
|
File without changes
|
@@ -1 +0,0 @@
|
|
1
|
-
f68518d3020fd6430af89b8a7e2261ff0bcc4e6325685baca3256f5f99bf51e8003353fa85ce95984de17cf3a19223bcd8d2cfa7df0fc07887ef937919493208
|
@@ -1,59 +0,0 @@
|
|
1
|
-
RSpec.describe "Auditing pipeline" do # rubocop:disable RSpec/DescribeClass
|
2
|
-
let(:topic) { TestTopic.new }
|
3
|
-
let(:config) do
|
4
|
-
NulogyMessageBusConsumer::Config.new(
|
5
|
-
consumer_group_id: random_consumer_group,
|
6
|
-
bootstrap_servers: test_bootstrap_servers,
|
7
|
-
topic_name: topic.topic_name
|
8
|
-
)
|
9
|
-
end
|
10
|
-
let(:logger) { spy }
|
11
|
-
|
12
|
-
after { topic.close }
|
13
|
-
|
14
|
-
context "when some messages have not been processed" do
|
15
|
-
it "logs the list of unprocessed messages" do
|
16
|
-
produce_message(id: uuid(1))
|
17
|
-
process_message(id: uuid(1))
|
18
|
-
produce_message(id: uuid(2))
|
19
|
-
|
20
|
-
expect(logger).to receive(:warn).with(include_json(
|
21
|
-
event: "unprocessed_message",
|
22
|
-
kafka_message: {id: uuid(2)}
|
23
|
-
))
|
24
|
-
|
25
|
-
run_audit_pipeline
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
context "when all messages have been processed" do
|
30
|
-
it "does not log anything" do
|
31
|
-
produce_message(id: uuid(1))
|
32
|
-
process_message(id: uuid(1))
|
33
|
-
|
34
|
-
run_audit_pipeline
|
35
|
-
|
36
|
-
expect(logger).to have_not_received(:warn)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def run_audit_pipeline
|
41
|
-
NulogyMessageBusConsumer
|
42
|
-
.consumer_audit_pipeline(config: config, logger: logger)
|
43
|
-
.invoke
|
44
|
-
end
|
45
|
-
|
46
|
-
def produce_message(id:)
|
47
|
-
topic.produce_one_message(
|
48
|
-
payload: JSON.dump(id: id)
|
49
|
-
)
|
50
|
-
end
|
51
|
-
|
52
|
-
def process_message(id:)
|
53
|
-
NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
|
54
|
-
end
|
55
|
-
|
56
|
-
def uuid(id)
|
57
|
-
format("00000000-0000-0000-0000-%012d", id)
|
58
|
-
end
|
59
|
-
end
|
@@ -1,41 +0,0 @@
|
|
1
|
-
RSpec.describe NulogyMessageBusConsumer::KafkaUtils do
|
2
|
-
subject(:utils) { NulogyMessageBusConsumer::KafkaUtils }
|
3
|
-
|
4
|
-
let(:topic) { TestTopic.new }
|
5
|
-
|
6
|
-
after { topic.close }
|
7
|
-
|
8
|
-
describe "#seek_beginning" do
|
9
|
-
it "updates the consumer offset to the beginning of the topic" do
|
10
|
-
topic.produce_one_message(payload: "First Message")
|
11
|
-
expect(topic.consume_one_message).to have_attributes(payload: "First Message")
|
12
|
-
expect(topic.consume_one_message).to eq(nil)
|
13
|
-
|
14
|
-
utils.seek_beginning(topic.consumer)
|
15
|
-
|
16
|
-
expect(topic.consume_one_message).to have_attributes(payload: "First Message")
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
describe "#seek_end" do
|
21
|
-
it "updates the consumer offset to the end of the topic" do
|
22
|
-
topic.produce_one_message(payload: "First Message")
|
23
|
-
|
24
|
-
utils.seek_ending(topic.consumer)
|
25
|
-
|
26
|
-
expect(topic.consume_one_message).to eq(nil)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
describe "#every_message_until_none_are_left" do
|
31
|
-
it "does not keep the connection open when there are no messages" do
|
32
|
-
topic.produce_one_message(payload: "The Only Message")
|
33
|
-
|
34
|
-
enum = utils.every_message_until_none_are_left(topic.consumer)
|
35
|
-
|
36
|
-
expect(enum).to match([
|
37
|
-
have_attributes(payload: "The Only Message")
|
38
|
-
])
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,131 +0,0 @@
|
|
1
|
-
RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
|
2
|
-
let(:test_topic) { TestTopic.new }
|
3
|
-
let(:consumer) { test_topic.consumer }
|
4
|
-
let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
|
5
|
-
let(:handler) { spy }
|
6
|
-
let(:pipeline) do
|
7
|
-
NulogyMessageBusConsumer::Pipeline.new([
|
8
|
-
NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
|
9
|
-
NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
|
10
|
-
NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
|
11
|
-
NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
|
12
|
-
handler
|
13
|
-
])
|
14
|
-
end
|
15
|
-
let(:deduped_pipeline) do
|
16
|
-
NulogyMessageBusConsumer::Pipeline.new([
|
17
|
-
NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
|
18
|
-
NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
|
19
|
-
NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
|
20
|
-
NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger),
|
21
|
-
NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
|
22
|
-
handler
|
23
|
-
])
|
24
|
-
end
|
25
|
-
|
26
|
-
after { test_topic.close }
|
27
|
-
|
28
|
-
context "when successful" do
|
29
|
-
it "commits and processes the next message" do
|
30
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
|
31
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
|
32
|
-
expect(consumer).to receive(:commit).twice
|
33
|
-
|
34
|
-
test_topic.produce_one_message(key: "test 1")
|
35
|
-
test_topic.produce_one_message(key: "test 2")
|
36
|
-
|
37
|
-
pipeline.invoke
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
context "when failing by :failure" do
|
42
|
-
it "reprocesses the message" do
|
43
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:failure)
|
44
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
|
45
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
|
46
|
-
expect(consumer).to receive(:commit).twice
|
47
|
-
|
48
|
-
test_topic.produce_one_message(key: "test 1")
|
49
|
-
test_topic.produce_one_message(key: "test 2")
|
50
|
-
|
51
|
-
pipeline.invoke
|
52
|
-
pipeline.invoke
|
53
|
-
end
|
54
|
-
|
55
|
-
# This test is more illustrative of how we expect it to work.
|
56
|
-
# Specifically, testing the "auto.offset.store" setting for the consumer.
|
57
|
-
context "when a partition has a failing message" do
|
58
|
-
let(:handler) { ->(message:, **_) { message.event_data[:type] == "good" ? :success : :failure } }
|
59
|
-
|
60
|
-
it "processes messages from other partitions without committing offsets for partitions with failing messages" do
|
61
|
-
Kafka.create_topic(test_topic.topic_name)
|
62
|
-
|
63
|
-
# Produce message to a single partition. This partition will be blocked by the second message.
|
64
|
-
test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # success
|
65
|
-
test_topic.produce_one_message(partition: 1, event_json: {type: "bad"}) # failure
|
66
|
-
blocked_id = test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # blocked
|
67
|
-
|
68
|
-
consume_from_partition(1) do
|
69
|
-
deduped_pipeline.invoke
|
70
|
-
end
|
71
|
-
|
72
|
-
# produce to another partition
|
73
|
-
success_id = test_topic.produce_one_message(partition: 2, event_json: {type: "good"}) # success
|
74
|
-
|
75
|
-
consume_from_partition(2) do
|
76
|
-
deduped_pipeline.invoke
|
77
|
-
end
|
78
|
-
|
79
|
-
# try consuming from all partitions again -- it will fail on the blocked one again
|
80
|
-
deduped_pipeline.invoke
|
81
|
-
|
82
|
-
# Wait for assignment after a reconnect
|
83
|
-
NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
|
84
|
-
|
85
|
-
lag = consumer.lag(consumer.committed)
|
86
|
-
expect(lag.dig(test_topic.topic_name, 1)).to be >= 1
|
87
|
-
expect(lag.dig(test_topic.topic_name, 2)).to be(0)
|
88
|
-
expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(success_id)).to be(true)
|
89
|
-
expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(blocked_id)).to be(false)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
context "when failing by exception" do
|
95
|
-
it "reprocesses the message" do
|
96
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_raise("intentional error")
|
97
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
|
98
|
-
expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
|
99
|
-
expect(consumer).to receive(:commit).twice
|
100
|
-
|
101
|
-
test_topic.produce_one_message(key: "test 1")
|
102
|
-
test_topic.produce_one_message(key: "test 2")
|
103
|
-
|
104
|
-
expect {
|
105
|
-
pipeline.invoke
|
106
|
-
}.to raise_error("intentional error")
|
107
|
-
|
108
|
-
pipeline.invoke
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
def a_message_with(matcher)
|
113
|
-
hash_including(
|
114
|
-
message: have_attributes(matcher)
|
115
|
-
)
|
116
|
-
end
|
117
|
-
|
118
|
-
def consume_from_partition(partition_number)
|
119
|
-
original_assignment = consumer.assignment
|
120
|
-
topic_partitions = original_assignment
|
121
|
-
.to_h
|
122
|
-
.transform_values { |values| values.select { |t| t.partition == partition_number } }
|
123
|
-
new_assignment = Rdkafka::Consumer::TopicPartitionList.new(topic_partitions)
|
124
|
-
|
125
|
-
consumer.assign(new_assignment)
|
126
|
-
|
127
|
-
yield
|
128
|
-
ensure
|
129
|
-
consumer.assign(original_assignment)
|
130
|
-
end
|
131
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
RSpec.describe NulogyMessageBusConsumer::Steps::ConnectToMessageBus do
|
2
|
-
subject(:pipeline) do
|
3
|
-
pipeline = NulogyMessageBusConsumer.recommended_consumer_pipeline(config: config)
|
4
|
-
pipeline.insert(tap, after: NulogyMessageBusConsumer::Steps::ConnectToMessageBus)
|
5
|
-
pipeline.append(message_handler_spy)
|
6
|
-
pipeline
|
7
|
-
end
|
8
|
-
|
9
|
-
let(:topic) { TestTopic.new }
|
10
|
-
let(:config) { topic.config }
|
11
|
-
let(:tap) { MiddlewareTap.new }
|
12
|
-
let(:message_handler_spy) { double }
|
13
|
-
|
14
|
-
after { topic.close }
|
15
|
-
|
16
|
-
# TODO: This spec is terribly flakey.
|
17
|
-
xit "receives messages" do
|
18
|
-
called = false
|
19
|
-
expect(message_handler_spy).to receive(:call) do |message:, **_kargs|
|
20
|
-
expect(message).to have_attributes(event_data: {data: "Some Payload"})
|
21
|
-
called = true
|
22
|
-
:success
|
23
|
-
end
|
24
|
-
|
25
|
-
pipeline_thread = start(pipeline, tap)
|
26
|
-
|
27
|
-
topic.produce_one_message(
|
28
|
-
key: "Some Key",
|
29
|
-
payload: message_payload(data: "Some Payload")
|
30
|
-
)
|
31
|
-
|
32
|
-
NulogyMessageBusConsumer::KafkaUtils.wait_for { called }
|
33
|
-
Thread.kill(pipeline_thread)
|
34
|
-
end
|
35
|
-
|
36
|
-
def start(pipeline, tap)
|
37
|
-
thr = Thread.new { pipeline.invoke }
|
38
|
-
wait_for_partition_assignment(tap)
|
39
|
-
thr
|
40
|
-
end
|
41
|
-
|
42
|
-
def wait_for_partition_assignment(tap)
|
43
|
-
NulogyMessageBusConsumer::KafkaUtils.wait_for { tap.arguments[:kafka_consumer] }
|
44
|
-
NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(tap.arguments[:kafka_consumer])
|
45
|
-
end
|
46
|
-
|
47
|
-
def message_payload(**payload)
|
48
|
-
JSON.dump(
|
49
|
-
id: SecureRandom.uuid,
|
50
|
-
created_at: 1_000,
|
51
|
-
event_json: JSON.dump(payload)
|
52
|
-
)
|
53
|
-
end
|
54
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
module NulogyMessageBusConsumer
|
2
|
-
module Steps
|
3
|
-
RSpec.describe SuperviseConsumerLag do
|
4
|
-
# RSpec does not like when objects (spies, doubles, etc) are used
|
5
|
-
# outside of its scope, e.g.
|
6
|
-
# The use of doubles or partial doubles from rspec-mocks outside of the per-test lifecycle is not supported. (RSpec::Mocks::OutsideOfExampleError)
|
7
|
-
let(:thread_spy) do
|
8
|
-
Class.new do
|
9
|
-
attr_reader :killed
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
@killed = false
|
13
|
-
end
|
14
|
-
|
15
|
-
def kill
|
16
|
-
@killed = true
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
it "kills the main thread after lag does not change" do
|
22
|
-
thread = thread_spy.new
|
23
|
-
logger = spy
|
24
|
-
|
25
|
-
consumer = instance_double(
|
26
|
-
Rdkafka::Consumer,
|
27
|
-
committed: nil,
|
28
|
-
lag: {}
|
29
|
-
)
|
30
|
-
# skip waiting for assignment
|
31
|
-
allow(KafkaUtils).to receive(:wait_for_assignment).with(consumer).and_return(nil)
|
32
|
-
tracker = instance_double(
|
33
|
-
LagTracker,
|
34
|
-
failing_checks: 3,
|
35
|
-
failing?: true,
|
36
|
-
failed: {"topic" => ["partition1", "partition2"]}
|
37
|
-
).as_null_object
|
38
|
-
|
39
|
-
supervisor = described_class.new(
|
40
|
-
logger,
|
41
|
-
tracker: tracker,
|
42
|
-
killable: thread
|
43
|
-
)
|
44
|
-
|
45
|
-
supervisor.call(kafka_consumer: consumer) {}
|
46
|
-
|
47
|
-
KafkaUtils.wait_for { thread.killed }
|
48
|
-
|
49
|
-
expect(logger).to have_received(:warn).with(/Assigned partition lag has not changed/)
|
50
|
-
expect(thread.killed).to be(true)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
RSpec.describe TestTopic do
|
2
|
-
let(:topic) { TestTopic.new }
|
3
|
-
|
4
|
-
after { topic.close }
|
5
|
-
|
6
|
-
context "smoke test for specs" do
|
7
|
-
it "publishes and receives messages" do
|
8
|
-
topic.produce_one_message(
|
9
|
-
key: "Some Key",
|
10
|
-
payload: "Some Payload"
|
11
|
-
)
|
12
|
-
|
13
|
-
message = topic.consume_one_message
|
14
|
-
expect(message).to have_attributes(
|
15
|
-
key: "Some Key",
|
16
|
-
payload: "Some Payload"
|
17
|
-
)
|
18
|
-
end
|
19
|
-
|
20
|
-
it "receives returns nil when no messages are received" do
|
21
|
-
consumer = topic.consumer
|
22
|
-
message = consumer.poll(1)
|
23
|
-
expect(message).to be(nil)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
context "spec helpers" do
|
28
|
-
it "creates topics" do
|
29
|
-
create_topic(topic.topic_name)
|
30
|
-
expect(list_topics).to include(topic.topic_name)
|
31
|
-
end
|
32
|
-
|
33
|
-
it "deletes topics" do
|
34
|
-
create_topic(topic.topic_name)
|
35
|
-
delete_topic(topic.topic_name)
|
36
|
-
expect(list_topics).not_to include(topic.topic_name)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
data/spec/spec_helper.rb
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
require "bundler/setup"
|
2
|
-
require "rspec/json_expectations"
|
3
|
-
ENV["RAILS_ENV"] ||= "test"
|
4
|
-
require File.expand_path("dummy/config/environment", __dir__)
|
5
|
-
abort("The Rails environment is running in production mode!") if Rails.env.production?
|
6
|
-
require "rspec/rails"
|
7
|
-
require "dotenv"
|
8
|
-
|
9
|
-
env_file = ENV["CI"] == "true" ? ".env.ci" : ".env.local"
|
10
|
-
raise "Expected #{env_file}" unless File.exist?(env_file)
|
11
|
-
|
12
|
-
Dotenv.load(env_file)
|
13
|
-
|
14
|
-
# Load RSpec helpers.
|
15
|
-
ENGINE_ROOT ||= File.expand_path("..", __dir__)
|
16
|
-
Dir[File.join(ENGINE_ROOT, "spec/support/**/*.rb")].sort.each { |f| require f }
|
17
|
-
|
18
|
-
# Load migrations from the dummy app.
|
19
|
-
# ActiveRecord::Migrator.migrations_paths = File.join(ENGINE_ROOT, 'spec/dummy/db/migrate')
|
20
|
-
begin
|
21
|
-
ActiveRecord::Migration.maintain_test_schema!
|
22
|
-
rescue ActiveRecord::PendingMigrationError => e
|
23
|
-
puts e.to_s.strip
|
24
|
-
exit 1
|
25
|
-
end
|
26
|
-
|
27
|
-
RSpec.configure do |config|
|
28
|
-
config.example_status_persistence_file_path = ".rspec_status"
|
29
|
-
|
30
|
-
# Disable RSpec exposing methods globally on `Module` and `main`
|
31
|
-
config.disable_monkey_patching!
|
32
|
-
|
33
|
-
config.use_transactional_fixtures = true
|
34
|
-
|
35
|
-
config.expect_with :rspec do |c|
|
36
|
-
c.syntax = :expect
|
37
|
-
c.include_chain_clauses_in_custom_matcher_descriptions = true
|
38
|
-
end
|
39
|
-
|
40
|
-
config.mock_with :rspec do |mocks|
|
41
|
-
mocks.verify_partial_doubles = true
|
42
|
-
end
|
43
|
-
|
44
|
-
config.shared_context_metadata_behavior = :apply_to_host_groups
|
45
|
-
config.filter_rails_from_backtrace!
|
46
|
-
config.use_transactional_fixtures = true
|
47
|
-
|
48
|
-
config.include(Kafka)
|
49
|
-
end
|