nulogy_message_bus_consumer 0.3.2 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +2 -3
- data/config/credentials/message-bus-us-east-1.key +1 -0
- data/config/credentials/message-bus-us-east-1.yml.enc +1 -0
- data/lib/nulogy_message_bus_consumer.rb +27 -6
- data/lib/nulogy_message_bus_consumer/clock.rb +13 -0
- data/lib/nulogy_message_bus_consumer/config.rb +18 -4
- data/lib/nulogy_message_bus_consumer/deployment/ecs.rb +23 -0
- data/lib/nulogy_message_bus_consumer/lag_tracker.rb +53 -0
- data/lib/nulogy_message_bus_consumer/message.rb +1 -1
- data/lib/nulogy_message_bus_consumer/null_logger.rb +6 -3
- data/lib/nulogy_message_bus_consumer/pipeline.rb +1 -1
- data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb +2 -1
- data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb +23 -9
- data/lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb +1 -1
- data/lib/nulogy_message_bus_consumer/steps/log_messages.rb +4 -11
- data/lib/nulogy_message_bus_consumer/steps/stream_messages.rb +2 -2
- data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb +2 -2
- data/lib/nulogy_message_bus_consumer/steps/timed_task.rb +42 -0
- data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb +45 -0
- data/lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb +37 -0
- data/lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb +65 -0
- data/lib/nulogy_message_bus_consumer/version.rb +1 -1
- data/lib/tasks/engine/message_bus_consumer.rake +7 -8
- data/spec/dummy/Rakefile +6 -0
- data/spec/dummy/app/assets/config/manifest.js +3 -0
- data/spec/dummy/app/assets/stylesheets/application.css +15 -0
- data/spec/dummy/app/channels/application_cable/channel.rb +4 -0
- data/spec/dummy/app/channels/application_cable/connection.rb +4 -0
- data/spec/dummy/app/controllers/application_controller.rb +2 -0
- data/spec/dummy/app/helpers/application_helper.rb +2 -0
- data/spec/dummy/app/javascript/packs/application.js +15 -0
- data/spec/dummy/app/jobs/application_job.rb +7 -0
- data/spec/dummy/app/mailers/application_mailer.rb +4 -0
- data/spec/dummy/app/models/application_record.rb +3 -0
- data/spec/dummy/app/views/layouts/application.html.erb +14 -0
- data/spec/dummy/app/views/layouts/mailer.html.erb +13 -0
- data/spec/dummy/app/views/layouts/mailer.text.erb +1 -0
- data/spec/dummy/bin/rails +4 -0
- data/spec/dummy/bin/rake +4 -0
- data/spec/dummy/bin/setup +33 -0
- data/spec/dummy/config.ru +5 -0
- data/spec/dummy/config/application.rb +29 -0
- data/spec/dummy/config/boot.rb +5 -0
- data/spec/dummy/config/cable.yml +10 -0
- data/spec/dummy/config/credentials/message-bus-us-east-1.key +1 -0
- data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc +1 -0
- data/spec/dummy/config/database.yml +27 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +62 -0
- data/spec/dummy/config/environments/production.rb +112 -0
- data/spec/dummy/config/environments/test.rb +49 -0
- data/spec/dummy/config/initializers/application_controller_renderer.rb +8 -0
- data/spec/dummy/config/initializers/assets.rb +12 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/content_security_policy.rb +28 -0
- data/spec/dummy/config/initializers/cookies_serializer.rb +5 -0
- data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
- data/spec/dummy/config/initializers/inflections.rb +16 -0
- data/spec/dummy/config/initializers/message_bus_consumer.rb +5 -0
- data/spec/dummy/config/initializers/mime_types.rb +4 -0
- data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/spec/dummy/config/locales/en.yml +33 -0
- data/spec/dummy/config/puma.rb +36 -0
- data/spec/dummy/config/routes.rb +3 -0
- data/spec/dummy/config/spring.rb +6 -0
- data/spec/dummy/config/storage.yml +34 -0
- data/spec/dummy/db/schema.rb +21 -0
- data/spec/dummy/log/development.log +4 -0
- data/spec/dummy/log/production.log +18 -0
- data/spec/dummy/log/test.log +7949 -0
- data/spec/dummy/public/404.html +67 -0
- data/spec/dummy/public/422.html +67 -0
- data/spec/dummy/public/500.html +66 -0
- data/spec/dummy/public/apple-touch-icon-precomposed.png +0 -0
- data/spec/dummy/public/apple-touch-icon.png +0 -0
- data/spec/dummy/public/favicon.ico +0 -0
- data/spec/dummy/tmp/development_secret.txt +1 -0
- data/spec/integration/nulogy_message_bus_consumer/auditor_spec.rb +59 -0
- data/spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb +41 -0
- data/spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +131 -0
- data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb +53 -0
- data/spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb +32 -0
- data/spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb +33 -0
- data/spec/integration/test_topic_spec.rb +39 -0
- data/spec/spec_helper.rb +50 -0
- data/spec/support/kafka.rb +74 -0
- data/spec/support/middleware_tap.rb +12 -0
- data/spec/support/skip.rb +9 -0
- data/spec/support/test_topic.rb +48 -0
- data/spec/unit/nulogy_message_bus_consumer/config_spec.rb +20 -0
- data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb +35 -0
- data/spec/unit/nulogy_message_bus_consumer/message_spec.rb +84 -0
- data/spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb +49 -0
- data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +58 -0
- data/spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb +56 -0
- data/spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb +70 -0
- data/spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb +35 -0
- data/spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb +67 -0
- data/spec/unit/nulogy_message_bus_consumer_spec.rb +30 -0
- metadata +209 -21
- data/lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb +0 -51
@@ -0,0 +1,49 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer::Pipeline do
|
2
|
+
let(:handler) { spy }
|
3
|
+
|
4
|
+
describe "#invoke_pipeline" do
|
5
|
+
it "raises if handler is not provided" do
|
6
|
+
expect { invoke_pipeline }.to raise_error("Handlers are the end of the line. Do not use yield.")
|
7
|
+
end
|
8
|
+
|
9
|
+
it "calls the handler with message" do
|
10
|
+
message = NulogyMessageBusConsumer::Message.new
|
11
|
+
|
12
|
+
expect(handler).to receive(:call).with(message: message)
|
13
|
+
|
14
|
+
invoke_pipeline(
|
15
|
+
->(**_, &block) { block.yield(message: message) },
|
16
|
+
handler
|
17
|
+
)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "passes information along" do
|
21
|
+
message = NulogyMessageBusConsumer::Message.new
|
22
|
+
|
23
|
+
expect(handler).to receive(:call).with(message: message)
|
24
|
+
|
25
|
+
invoke_pipeline(
|
26
|
+
->(**_, &block) { block.yield(message: message) },
|
27
|
+
->(**_, &block) { block.yield },
|
28
|
+
handler
|
29
|
+
)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "raises when overwriting an existing key" do
|
33
|
+
message = NulogyMessageBusConsumer::Message.new
|
34
|
+
other_message = NulogyMessageBusConsumer::Message.new
|
35
|
+
|
36
|
+
expect {
|
37
|
+
invoke_pipeline(
|
38
|
+
->(**_, &block) { block.yield(message: message) },
|
39
|
+
->(**_, &block) { block.yield(message: other_message) },
|
40
|
+
handler
|
41
|
+
)
|
42
|
+
}.to raise_error("Cannot override existing argument(s): message")
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def invoke_pipeline(*steps)
|
47
|
+
NulogyMessageBusConsumer::Pipeline.new(steps).invoke
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
|
2
|
+
subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new }
|
3
|
+
|
4
|
+
let(:kafka_consumer) { spy }
|
5
|
+
let(:message) { NulogyMessageBusConsumer::Message.new }
|
6
|
+
|
7
|
+
context "when result is :success" do
|
8
|
+
it "returns the result" do
|
9
|
+
result = step.call(kafka_consumer: kafka_consumer, message: message) { :success }
|
10
|
+
|
11
|
+
expect(result).to be(:success)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "commits the message offset" do
|
15
|
+
expect(kafka_consumer).to receive(:store_offset).with(message)
|
16
|
+
expect(kafka_consumer).to receive(:commit)
|
17
|
+
|
18
|
+
step.call(kafka_consumer: kafka_consumer, message: message) { :success }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context "when result is :failure" do
|
23
|
+
it "returns the result" do
|
24
|
+
result = step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
|
25
|
+
|
26
|
+
expect(result).to be(:failure)
|
27
|
+
end
|
28
|
+
|
29
|
+
it "reconnects to Kafka when failure" do
|
30
|
+
# When subscribing, we subscribe to a topic (i.e. the keys)
|
31
|
+
topic_partition_list = {topic_name: %i[partition_1 partition_2]}
|
32
|
+
allow(kafka_consumer).to receive(:subscription).and_return(topic_partition_list)
|
33
|
+
|
34
|
+
expect(kafka_consumer).to receive(:unsubscribe)
|
35
|
+
expect(kafka_consumer).to receive(:subscribe).with(:topic_name)
|
36
|
+
|
37
|
+
step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context "when result is something else" do
|
42
|
+
it "raises" do
|
43
|
+
expect {
|
44
|
+
step.call(kafka_consumer: kafka_consumer, message: message) { :bogus_value }
|
45
|
+
}.to raise_error("'bogus_value' is not a valid processing outcome. Must be :success or :failure")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
context "when downstream steps raise" do
|
50
|
+
it "does not commit" do
|
51
|
+
expect(kafka_consumer).not_to receive(:commit)
|
52
|
+
|
53
|
+
expect {
|
54
|
+
step.call(kafka_consumer: kafka_consumer, message: message) { raise "oopsie" }
|
55
|
+
}.to raise_error("oopsie")
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer::Steps::DeduplicateMessages do
|
2
|
+
subject(:deduplicate_messages) { NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger) }
|
3
|
+
|
4
|
+
let(:logger) { spy }
|
5
|
+
let(:id) { SecureRandom.uuid }
|
6
|
+
let(:message) { NulogyMessageBusConsumer::Message.new(id: id) }
|
7
|
+
|
8
|
+
context "when a duplicate message is consumed" do
|
9
|
+
it "logs a duplicate was detected" do
|
10
|
+
expect(logger).to receive(:warn).with(include_json(
|
11
|
+
event: "duplicate_message_detected",
|
12
|
+
kafka_message_id: id
|
13
|
+
))
|
14
|
+
|
15
|
+
deduplicate_messages.call(message: message) { :success }
|
16
|
+
deduplicate_messages.call(message: message) { raise "Not evaluated" }
|
17
|
+
end
|
18
|
+
|
19
|
+
it "returns :success" do
|
20
|
+
deduplicate_messages.call(message: message) { :success }
|
21
|
+
|
22
|
+
result = deduplicate_messages.call(message: message) { raise "Not evaluated" }
|
23
|
+
|
24
|
+
expect(result).to be(:success)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context "when a duplicate message is processed at the same time" do
|
29
|
+
it "logs a duplicate was detected" do
|
30
|
+
expect(logger).to receive(:warn).with(include_json(
|
31
|
+
event: "duplicate_message_detected",
|
32
|
+
kafka_message_id: id
|
33
|
+
))
|
34
|
+
|
35
|
+
deduplicate_messages.call(message: message) do
|
36
|
+
NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
|
37
|
+
:success
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
it "returns :success" do
|
42
|
+
result = deduplicate_messages.call(message: message) {
|
43
|
+
NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
|
44
|
+
:success
|
45
|
+
}
|
46
|
+
|
47
|
+
expect(result).to be(:success)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
it "records message was processed" do
|
52
|
+
deduplicate_messages.call(message: message) { :success }
|
53
|
+
|
54
|
+
expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(id: id)).to be(true)
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer::Steps::LogMessages do
|
2
|
+
let(:logger) { spy }
|
3
|
+
|
4
|
+
it "logs the message was received" do
|
5
|
+
log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
|
6
|
+
|
7
|
+
expect(logger).to receive(:info).with(include_json(
|
8
|
+
event: "message_received",
|
9
|
+
kafka_message_id: "id",
|
10
|
+
message: "Received id"
|
11
|
+
))
|
12
|
+
|
13
|
+
log_message.call(message: message) {}
|
14
|
+
end
|
15
|
+
|
16
|
+
it "logs the message was processed" do
|
17
|
+
log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
|
18
|
+
|
19
|
+
expect(logger).to receive(:info).with(include_json(
|
20
|
+
event: "message_processed",
|
21
|
+
kafka_message_id: "id",
|
22
|
+
message: match(/^Processed id/)
|
23
|
+
))
|
24
|
+
|
25
|
+
log_message.call(message: message) {}
|
26
|
+
end
|
27
|
+
|
28
|
+
it "logs the message's time to processed" do
|
29
|
+
# clock always gives milliseconds since epoch
|
30
|
+
clock = instance_double(NulogyMessageBusConsumer::Clock, ms: 1_000_001_234)
|
31
|
+
log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger, clock: clock)
|
32
|
+
|
33
|
+
expect(logger).to receive(:info).with(include_json(
|
34
|
+
event: "message_processed",
|
35
|
+
kafka_message_id: "id",
|
36
|
+
time_to_processed: 1234
|
37
|
+
))
|
38
|
+
|
39
|
+
# debezium converts time with zone to nanoseconds since epoch
|
40
|
+
log_message.call(message: message(created_at: 1_000_000_000_000)) {}
|
41
|
+
end
|
42
|
+
|
43
|
+
it "logs the message's result" do
|
44
|
+
log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
|
45
|
+
|
46
|
+
expect(logger).to receive(:info).with(include_json(
|
47
|
+
event: "message_processed",
|
48
|
+
result: "success"
|
49
|
+
))
|
50
|
+
|
51
|
+
log_message.call(message: message) { :success }
|
52
|
+
|
53
|
+
expect(logger).to receive(:info).with(include_json(
|
54
|
+
event: "message_processed",
|
55
|
+
result: "failure"
|
56
|
+
))
|
57
|
+
|
58
|
+
log_message.call(message: message) { :failure }
|
59
|
+
end
|
60
|
+
|
61
|
+
it "returns a result" do
|
62
|
+
log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
|
63
|
+
|
64
|
+
expect(log_message.call(message: message) { :return_value }).to eq(:return_value)
|
65
|
+
end
|
66
|
+
|
67
|
+
def message(id: "id", created_at: 1_000, **attrs)
|
68
|
+
NulogyMessageBusConsumer::Message.new(id: id, created_at: created_at, **attrs)
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer::Steps::StreamMessages do
|
2
|
+
subject(:step) { NulogyMessageBusConsumer::Steps::StreamMessages.new(logger) }
|
3
|
+
|
4
|
+
let(:config) { NulogyMessageBusConsumer::Config.new }
|
5
|
+
let(:logger) { spy }
|
6
|
+
|
7
|
+
it "logs errors while streaming from kafka" do
|
8
|
+
kafka_consumer = instance_double("Rdkafka::Consumer")
|
9
|
+
allow(kafka_consumer).to receive(:subscribe)
|
10
|
+
allow(kafka_consumer).to receive(:unsubscribe)
|
11
|
+
allow(kafka_consumer).to receive(:each).and_raise(StandardError, "streaming failed")
|
12
|
+
|
13
|
+
expect(logger).to receive(:error).with(include_json(
|
14
|
+
event: "message_processing_errored",
|
15
|
+
class: "StandardError",
|
16
|
+
message: "streaming failed"
|
17
|
+
))
|
18
|
+
|
19
|
+
expect {
|
20
|
+
step.call(kafka_consumer: kafka_consumer)
|
21
|
+
}.to raise_error(StandardError)
|
22
|
+
end
|
23
|
+
|
24
|
+
context "when handler returns failure" do
|
25
|
+
it "does not commit the message in kafka" do
|
26
|
+
kafka_consumer = spy
|
27
|
+
allow(kafka_consumer).to receive(:each).and_yield(anything)
|
28
|
+
allow(NulogyMessageBusConsumer::Message).to receive(:from_kafka).and_return(NulogyMessageBusConsumer::Message.new)
|
29
|
+
|
30
|
+
expect(kafka_consumer).not_to receive(:commit)
|
31
|
+
|
32
|
+
step.call(kafka_consumer: kafka_consumer) { :failure }
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module NulogyMessageBusConsumer
|
2
|
+
module Tasks
|
3
|
+
RSpec.describe LogConsumerLag::Calculator do
|
4
|
+
subject(:calculator) { LogConsumerLag::Calculator }
|
5
|
+
|
6
|
+
describe ".add_max_lag" do
|
7
|
+
it "adds max lag per partition" do
|
8
|
+
lag_per_topic = {
|
9
|
+
topic_1: {
|
10
|
+
partition_1: 100,
|
11
|
+
partition_2: 900,
|
12
|
+
partition_3: 100
|
13
|
+
},
|
14
|
+
topic_2: {
|
15
|
+
partition_1: 100,
|
16
|
+
partition_2: 200,
|
17
|
+
partition_3: 100
|
18
|
+
}
|
19
|
+
}
|
20
|
+
|
21
|
+
lag_with_max = calculator.add_max_lag(lag_per_topic)
|
22
|
+
|
23
|
+
expect(lag_with_max).to match(
|
24
|
+
_max: 900,
|
25
|
+
topic_1: {
|
26
|
+
_max: 900,
|
27
|
+
partition_1: 100,
|
28
|
+
partition_2: 900,
|
29
|
+
partition_3: 100
|
30
|
+
},
|
31
|
+
topic_2: {
|
32
|
+
_max: 200,
|
33
|
+
partition_1: 100,
|
34
|
+
partition_2: 200,
|
35
|
+
partition_3: 100
|
36
|
+
}
|
37
|
+
)
|
38
|
+
end
|
39
|
+
|
40
|
+
it "sets the max to 0 when no partitions are subscribed to" do
|
41
|
+
lag_per_topic = {
|
42
|
+
topic_1: {}
|
43
|
+
}
|
44
|
+
|
45
|
+
lag_with_max = calculator.add_max_lag(lag_per_topic)
|
46
|
+
|
47
|
+
expect(lag_with_max).to match(
|
48
|
+
_max: 0,
|
49
|
+
topic_1: {
|
50
|
+
_max: 0
|
51
|
+
}
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
it "sets the max to 0 when no topics are subscribed to" do
|
56
|
+
lag_per_topic = {}
|
57
|
+
|
58
|
+
lag_with_max = calculator.add_max_lag(lag_per_topic)
|
59
|
+
|
60
|
+
expect(lag_with_max).to match(
|
61
|
+
_max: 0
|
62
|
+
)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
RSpec.describe NulogyMessageBusConsumer do
|
2
|
+
describe "#configure" do
|
3
|
+
before do
|
4
|
+
config = NulogyMessageBusConsumer.config
|
5
|
+
config.instance_variable_names.each do |variable_name|
|
6
|
+
config.remove_instance_variable(variable_name)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
it "allows configuration to be provided as arguments" do
|
11
|
+
NulogyMessageBusConsumer.configure(
|
12
|
+
bootstrap_servers: "some servers"
|
13
|
+
)
|
14
|
+
|
15
|
+
expect(NulogyMessageBusConsumer.config).to have_attributes(
|
16
|
+
bootstrap_servers: "some servers"
|
17
|
+
)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "allows configuration to be provided as a block" do
|
21
|
+
NulogyMessageBusConsumer.configure do |config|
|
22
|
+
config.bootstrap_servers = "some servers"
|
23
|
+
end
|
24
|
+
|
25
|
+
expect(NulogyMessageBusConsumer.config).to have_attributes(
|
26
|
+
bootstrap_servers: "some servers"
|
27
|
+
)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nulogy_message_bus_consumer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nulogy
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -52,6 +52,34 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler-audit
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.7.0.1
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.7.0.1
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: dotenv
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 2.7.6
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 2.7.6
|
55
83
|
- !ruby/object:Gem::Dependency
|
56
84
|
name: pg
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,42 +128,42 @@ dependencies:
|
|
100
128
|
requirements:
|
101
129
|
- - '='
|
102
130
|
- !ruby/object:Gem::Version
|
103
|
-
version: 6.0.3
|
131
|
+
version: 6.0.3.5
|
104
132
|
type: :development
|
105
133
|
prerelease: false
|
106
134
|
version_requirements: !ruby/object:Gem::Requirement
|
107
135
|
requirements:
|
108
136
|
- - '='
|
109
137
|
- !ruby/object:Gem::Version
|
110
|
-
version: 6.0.3
|
138
|
+
version: 6.0.3.5
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: rake-release
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
114
142
|
requirements:
|
115
143
|
- - '='
|
116
144
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.
|
145
|
+
version: 1.3.0
|
118
146
|
type: :development
|
119
147
|
prerelease: false
|
120
148
|
version_requirements: !ruby/object:Gem::Requirement
|
121
149
|
requirements:
|
122
150
|
- - '='
|
123
151
|
- !ruby/object:Gem::Version
|
124
|
-
version: 1.
|
152
|
+
version: 1.3.0
|
125
153
|
- !ruby/object:Gem::Dependency
|
126
154
|
name: rspec
|
127
155
|
requirement: !ruby/object:Gem::Requirement
|
128
156
|
requirements:
|
129
|
-
- -
|
157
|
+
- - "~>"
|
130
158
|
- !ruby/object:Gem::Version
|
131
|
-
version: 3.
|
159
|
+
version: '3.10'
|
132
160
|
type: :development
|
133
161
|
prerelease: false
|
134
162
|
version_requirements: !ruby/object:Gem::Requirement
|
135
163
|
requirements:
|
136
|
-
- -
|
164
|
+
- - "~>"
|
137
165
|
- !ruby/object:Gem::Version
|
138
|
-
version: 3.
|
166
|
+
version: '3.10'
|
139
167
|
- !ruby/object:Gem::Dependency
|
140
168
|
name: rspec-json_expectations
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -165,19 +193,19 @@ dependencies:
|
|
165
193
|
- !ruby/object:Gem::Version
|
166
194
|
version: 4.0.1
|
167
195
|
- !ruby/object:Gem::Dependency
|
168
|
-
name:
|
196
|
+
name: standard
|
169
197
|
requirement: !ruby/object:Gem::Requirement
|
170
198
|
requirements:
|
171
199
|
- - '='
|
172
200
|
- !ruby/object:Gem::Version
|
173
|
-
version: 0.
|
201
|
+
version: 0.11.0
|
174
202
|
type: :development
|
175
203
|
prerelease: false
|
176
204
|
version_requirements: !ruby/object:Gem::Requirement
|
177
205
|
requirements:
|
178
206
|
- - '='
|
179
207
|
- !ruby/object:Gem::Version
|
180
|
-
version: 0.
|
208
|
+
version: 0.11.0
|
181
209
|
- !ruby/object:Gem::Dependency
|
182
210
|
name: rubocop-rails
|
183
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,7 +234,7 @@ dependencies:
|
|
206
234
|
- - '='
|
207
235
|
- !ruby/object:Gem::Version
|
208
236
|
version: 1.38.1
|
209
|
-
description:
|
237
|
+
description:
|
210
238
|
email:
|
211
239
|
- tass@nulogy.com
|
212
240
|
executables: []
|
@@ -214,13 +242,18 @@ extensions: []
|
|
214
242
|
extra_rdoc_files: []
|
215
243
|
files:
|
216
244
|
- Rakefile
|
245
|
+
- config/credentials/message-bus-us-east-1.key
|
246
|
+
- config/credentials/message-bus-us-east-1.yml.enc
|
217
247
|
- config/routes.rb
|
218
248
|
- db/migrate/20200509095105_create_message_bus_processed_messages.rb
|
219
249
|
- lib/nulogy_message_bus_consumer.rb
|
250
|
+
- lib/nulogy_message_bus_consumer/clock.rb
|
220
251
|
- lib/nulogy_message_bus_consumer/config.rb
|
252
|
+
- lib/nulogy_message_bus_consumer/deployment/ecs.rb
|
221
253
|
- lib/nulogy_message_bus_consumer/engine.rb
|
222
254
|
- lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb
|
223
255
|
- lib/nulogy_message_bus_consumer/kafka_utils.rb
|
256
|
+
- lib/nulogy_message_bus_consumer/lag_tracker.rb
|
224
257
|
- lib/nulogy_message_bus_consumer/message.rb
|
225
258
|
- lib/nulogy_message_bus_consumer/null_logger.rb
|
226
259
|
- lib/nulogy_message_bus_consumer/pipeline.rb
|
@@ -229,17 +262,96 @@ files:
|
|
229
262
|
- lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
|
230
263
|
- lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
|
231
264
|
- lib/nulogy_message_bus_consumer/steps/log_messages.rb
|
232
|
-
- lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb
|
233
265
|
- lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb
|
234
266
|
- lib/nulogy_message_bus_consumer/steps/stream_messages.rb
|
235
267
|
- lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb
|
268
|
+
- lib/nulogy_message_bus_consumer/steps/timed_task.rb
|
269
|
+
- lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb
|
270
|
+
- lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb
|
271
|
+
- lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb
|
236
272
|
- lib/nulogy_message_bus_consumer/version.rb
|
237
273
|
- lib/tasks/engine/message_bus_consumer.rake
|
274
|
+
- spec/dummy/Rakefile
|
275
|
+
- spec/dummy/app/assets/config/manifest.js
|
276
|
+
- spec/dummy/app/assets/stylesheets/application.css
|
277
|
+
- spec/dummy/app/channels/application_cable/channel.rb
|
278
|
+
- spec/dummy/app/channels/application_cable/connection.rb
|
279
|
+
- spec/dummy/app/controllers/application_controller.rb
|
280
|
+
- spec/dummy/app/helpers/application_helper.rb
|
281
|
+
- spec/dummy/app/javascript/packs/application.js
|
282
|
+
- spec/dummy/app/jobs/application_job.rb
|
283
|
+
- spec/dummy/app/mailers/application_mailer.rb
|
284
|
+
- spec/dummy/app/models/application_record.rb
|
285
|
+
- spec/dummy/app/views/layouts/application.html.erb
|
286
|
+
- spec/dummy/app/views/layouts/mailer.html.erb
|
287
|
+
- spec/dummy/app/views/layouts/mailer.text.erb
|
288
|
+
- spec/dummy/bin/rails
|
289
|
+
- spec/dummy/bin/rake
|
290
|
+
- spec/dummy/bin/setup
|
291
|
+
- spec/dummy/config.ru
|
292
|
+
- spec/dummy/config/application.rb
|
293
|
+
- spec/dummy/config/boot.rb
|
294
|
+
- spec/dummy/config/cable.yml
|
295
|
+
- spec/dummy/config/credentials/message-bus-us-east-1.key
|
296
|
+
- spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
|
297
|
+
- spec/dummy/config/database.yml
|
298
|
+
- spec/dummy/config/environment.rb
|
299
|
+
- spec/dummy/config/environments/development.rb
|
300
|
+
- spec/dummy/config/environments/production.rb
|
301
|
+
- spec/dummy/config/environments/test.rb
|
302
|
+
- spec/dummy/config/initializers/application_controller_renderer.rb
|
303
|
+
- spec/dummy/config/initializers/assets.rb
|
304
|
+
- spec/dummy/config/initializers/backtrace_silencers.rb
|
305
|
+
- spec/dummy/config/initializers/content_security_policy.rb
|
306
|
+
- spec/dummy/config/initializers/cookies_serializer.rb
|
307
|
+
- spec/dummy/config/initializers/filter_parameter_logging.rb
|
308
|
+
- spec/dummy/config/initializers/inflections.rb
|
309
|
+
- spec/dummy/config/initializers/message_bus_consumer.rb
|
310
|
+
- spec/dummy/config/initializers/mime_types.rb
|
311
|
+
- spec/dummy/config/initializers/wrap_parameters.rb
|
312
|
+
- spec/dummy/config/locales/en.yml
|
313
|
+
- spec/dummy/config/puma.rb
|
314
|
+
- spec/dummy/config/routes.rb
|
315
|
+
- spec/dummy/config/spring.rb
|
316
|
+
- spec/dummy/config/storage.yml
|
317
|
+
- spec/dummy/db/schema.rb
|
318
|
+
- spec/dummy/log/development.log
|
319
|
+
- spec/dummy/log/production.log
|
320
|
+
- spec/dummy/log/test.log
|
321
|
+
- spec/dummy/public/404.html
|
322
|
+
- spec/dummy/public/422.html
|
323
|
+
- spec/dummy/public/500.html
|
324
|
+
- spec/dummy/public/apple-touch-icon-precomposed.png
|
325
|
+
- spec/dummy/public/apple-touch-icon.png
|
326
|
+
- spec/dummy/public/favicon.ico
|
327
|
+
- spec/dummy/tmp/development_secret.txt
|
328
|
+
- spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
|
329
|
+
- spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
|
330
|
+
- spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
|
331
|
+
- spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
|
332
|
+
- spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
|
333
|
+
- spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
|
334
|
+
- spec/integration/test_topic_spec.rb
|
335
|
+
- spec/spec_helper.rb
|
336
|
+
- spec/support/kafka.rb
|
337
|
+
- spec/support/middleware_tap.rb
|
338
|
+
- spec/support/skip.rb
|
339
|
+
- spec/support/test_topic.rb
|
340
|
+
- spec/unit/nulogy_message_bus_consumer/config_spec.rb
|
341
|
+
- spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
|
342
|
+
- spec/unit/nulogy_message_bus_consumer/message_spec.rb
|
343
|
+
- spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
|
344
|
+
- spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
|
345
|
+
- spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
|
346
|
+
- spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
|
347
|
+
- spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
|
348
|
+
- spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
|
349
|
+
- spec/unit/nulogy_message_bus_consumer_spec.rb
|
238
350
|
homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
|
239
351
|
licenses: []
|
240
352
|
metadata:
|
241
353
|
allowed_push_host: https://rubygems.org/
|
242
|
-
post_install_message:
|
354
|
+
post_install_message:
|
243
355
|
rdoc_options: []
|
244
356
|
require_paths:
|
245
357
|
- lib
|
@@ -247,15 +359,91 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
247
359
|
requirements:
|
248
360
|
- - ">="
|
249
361
|
- !ruby/object:Gem::Version
|
250
|
-
version: '
|
362
|
+
version: '2.6'
|
251
363
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
252
364
|
requirements:
|
253
365
|
- - ">="
|
254
366
|
- !ruby/object:Gem::Version
|
255
367
|
version: '0'
|
256
368
|
requirements: []
|
257
|
-
rubygems_version: 3.
|
258
|
-
signing_key:
|
369
|
+
rubygems_version: 3.2.15
|
370
|
+
signing_key:
|
259
371
|
specification_version: 4
|
260
372
|
summary: Code for accessing the Nulogy Message Bus
|
261
|
-
test_files:
|
373
|
+
test_files:
|
374
|
+
- spec/dummy/Rakefile
|
375
|
+
- spec/dummy/app/assets/config/manifest.js
|
376
|
+
- spec/dummy/app/assets/stylesheets/application.css
|
377
|
+
- spec/dummy/app/channels/application_cable/channel.rb
|
378
|
+
- spec/dummy/app/channels/application_cable/connection.rb
|
379
|
+
- spec/dummy/app/controllers/application_controller.rb
|
380
|
+
- spec/dummy/app/helpers/application_helper.rb
|
381
|
+
- spec/dummy/app/javascript/packs/application.js
|
382
|
+
- spec/dummy/app/jobs/application_job.rb
|
383
|
+
- spec/dummy/app/mailers/application_mailer.rb
|
384
|
+
- spec/dummy/app/models/application_record.rb
|
385
|
+
- spec/dummy/app/views/layouts/application.html.erb
|
386
|
+
- spec/dummy/app/views/layouts/mailer.html.erb
|
387
|
+
- spec/dummy/app/views/layouts/mailer.text.erb
|
388
|
+
- spec/dummy/bin/rails
|
389
|
+
- spec/dummy/bin/rake
|
390
|
+
- spec/dummy/bin/setup
|
391
|
+
- spec/dummy/config/application.rb
|
392
|
+
- spec/dummy/config/boot.rb
|
393
|
+
- spec/dummy/config/cable.yml
|
394
|
+
- spec/dummy/config/credentials/message-bus-us-east-1.key
|
395
|
+
- spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
|
396
|
+
- spec/dummy/config/database.yml
|
397
|
+
- spec/dummy/config/environment.rb
|
398
|
+
- spec/dummy/config/environments/development.rb
|
399
|
+
- spec/dummy/config/environments/production.rb
|
400
|
+
- spec/dummy/config/environments/test.rb
|
401
|
+
- spec/dummy/config/initializers/application_controller_renderer.rb
|
402
|
+
- spec/dummy/config/initializers/assets.rb
|
403
|
+
- spec/dummy/config/initializers/backtrace_silencers.rb
|
404
|
+
- spec/dummy/config/initializers/content_security_policy.rb
|
405
|
+
- spec/dummy/config/initializers/cookies_serializer.rb
|
406
|
+
- spec/dummy/config/initializers/filter_parameter_logging.rb
|
407
|
+
- spec/dummy/config/initializers/inflections.rb
|
408
|
+
- spec/dummy/config/initializers/message_bus_consumer.rb
|
409
|
+
- spec/dummy/config/initializers/mime_types.rb
|
410
|
+
- spec/dummy/config/initializers/wrap_parameters.rb
|
411
|
+
- spec/dummy/config/locales/en.yml
|
412
|
+
- spec/dummy/config/puma.rb
|
413
|
+
- spec/dummy/config/routes.rb
|
414
|
+
- spec/dummy/config/spring.rb
|
415
|
+
- spec/dummy/config/storage.yml
|
416
|
+
- spec/dummy/config.ru
|
417
|
+
- spec/dummy/db/schema.rb
|
418
|
+
- spec/dummy/log/development.log
|
419
|
+
- spec/dummy/log/production.log
|
420
|
+
- spec/dummy/log/test.log
|
421
|
+
- spec/dummy/public/404.html
|
422
|
+
- spec/dummy/public/422.html
|
423
|
+
- spec/dummy/public/500.html
|
424
|
+
- spec/dummy/public/apple-touch-icon-precomposed.png
|
425
|
+
- spec/dummy/public/apple-touch-icon.png
|
426
|
+
- spec/dummy/public/favicon.ico
|
427
|
+
- spec/dummy/tmp/development_secret.txt
|
428
|
+
- spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
|
429
|
+
- spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
|
430
|
+
- spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
|
431
|
+
- spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
|
432
|
+
- spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
|
433
|
+
- spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
|
434
|
+
- spec/integration/test_topic_spec.rb
|
435
|
+
- spec/spec_helper.rb
|
436
|
+
- spec/support/kafka.rb
|
437
|
+
- spec/support/middleware_tap.rb
|
438
|
+
- spec/support/skip.rb
|
439
|
+
- spec/support/test_topic.rb
|
440
|
+
- spec/unit/nulogy_message_bus_consumer/config_spec.rb
|
441
|
+
- spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
|
442
|
+
- spec/unit/nulogy_message_bus_consumer/message_spec.rb
|
443
|
+
- spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
|
444
|
+
- spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
|
445
|
+
- spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
|
446
|
+
- spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
|
447
|
+
- spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
|
448
|
+
- spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
|
449
|
+
- spec/unit/nulogy_message_bus_consumer_spec.rb
|