nulogy_message_bus_consumer 0.3.2 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +2 -3
  3. data/config/credentials/message-bus-us-east-1.key +1 -0
  4. data/config/credentials/message-bus-us-east-1.yml.enc +1 -0
  5. data/lib/nulogy_message_bus_consumer.rb +27 -6
  6. data/lib/nulogy_message_bus_consumer/clock.rb +13 -0
  7. data/lib/nulogy_message_bus_consumer/config.rb +18 -4
  8. data/lib/nulogy_message_bus_consumer/deployment/ecs.rb +23 -0
  9. data/lib/nulogy_message_bus_consumer/lag_tracker.rb +53 -0
  10. data/lib/nulogy_message_bus_consumer/message.rb +1 -1
  11. data/lib/nulogy_message_bus_consumer/null_logger.rb +6 -3
  12. data/lib/nulogy_message_bus_consumer/pipeline.rb +1 -1
  13. data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb +2 -1
  14. data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb +23 -9
  15. data/lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb +1 -1
  16. data/lib/nulogy_message_bus_consumer/steps/log_messages.rb +4 -11
  17. data/lib/nulogy_message_bus_consumer/steps/stream_messages.rb +2 -2
  18. data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb +2 -2
  19. data/lib/nulogy_message_bus_consumer/steps/timed_task.rb +42 -0
  20. data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb +45 -0
  21. data/lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb +37 -0
  22. data/lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb +65 -0
  23. data/lib/nulogy_message_bus_consumer/version.rb +1 -1
  24. data/lib/tasks/engine/message_bus_consumer.rake +7 -8
  25. data/spec/dummy/Rakefile +6 -0
  26. data/spec/dummy/app/assets/config/manifest.js +3 -0
  27. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  28. data/spec/dummy/app/channels/application_cable/channel.rb +4 -0
  29. data/spec/dummy/app/channels/application_cable/connection.rb +4 -0
  30. data/spec/dummy/app/controllers/application_controller.rb +2 -0
  31. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  32. data/spec/dummy/app/javascript/packs/application.js +15 -0
  33. data/spec/dummy/app/jobs/application_job.rb +7 -0
  34. data/spec/dummy/app/mailers/application_mailer.rb +4 -0
  35. data/spec/dummy/app/models/application_record.rb +3 -0
  36. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  37. data/spec/dummy/app/views/layouts/mailer.html.erb +13 -0
  38. data/spec/dummy/app/views/layouts/mailer.text.erb +1 -0
  39. data/spec/dummy/bin/rails +4 -0
  40. data/spec/dummy/bin/rake +4 -0
  41. data/spec/dummy/bin/setup +33 -0
  42. data/spec/dummy/config.ru +5 -0
  43. data/spec/dummy/config/application.rb +29 -0
  44. data/spec/dummy/config/boot.rb +5 -0
  45. data/spec/dummy/config/cable.yml +10 -0
  46. data/spec/dummy/config/credentials/message-bus-us-east-1.key +1 -0
  47. data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc +1 -0
  48. data/spec/dummy/config/database.yml +27 -0
  49. data/spec/dummy/config/environment.rb +5 -0
  50. data/spec/dummy/config/environments/development.rb +62 -0
  51. data/spec/dummy/config/environments/production.rb +112 -0
  52. data/spec/dummy/config/environments/test.rb +49 -0
  53. data/spec/dummy/config/initializers/application_controller_renderer.rb +8 -0
  54. data/spec/dummy/config/initializers/assets.rb +12 -0
  55. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  56. data/spec/dummy/config/initializers/content_security_policy.rb +28 -0
  57. data/spec/dummy/config/initializers/cookies_serializer.rb +5 -0
  58. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  59. data/spec/dummy/config/initializers/inflections.rb +16 -0
  60. data/spec/dummy/config/initializers/message_bus_consumer.rb +5 -0
  61. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  62. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  63. data/spec/dummy/config/locales/en.yml +33 -0
  64. data/spec/dummy/config/puma.rb +36 -0
  65. data/spec/dummy/config/routes.rb +3 -0
  66. data/spec/dummy/config/spring.rb +6 -0
  67. data/spec/dummy/config/storage.yml +34 -0
  68. data/spec/dummy/db/schema.rb +21 -0
  69. data/spec/dummy/log/development.log +4 -0
  70. data/spec/dummy/log/production.log +18 -0
  71. data/spec/dummy/log/test.log +7949 -0
  72. data/spec/dummy/public/404.html +67 -0
  73. data/spec/dummy/public/422.html +67 -0
  74. data/spec/dummy/public/500.html +66 -0
  75. data/spec/dummy/public/apple-touch-icon-precomposed.png +0 -0
  76. data/spec/dummy/public/apple-touch-icon.png +0 -0
  77. data/spec/dummy/public/favicon.ico +0 -0
  78. data/spec/dummy/tmp/development_secret.txt +1 -0
  79. data/spec/integration/nulogy_message_bus_consumer/auditor_spec.rb +59 -0
  80. data/spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb +41 -0
  81. data/spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +131 -0
  82. data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb +53 -0
  83. data/spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb +32 -0
  84. data/spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb +33 -0
  85. data/spec/integration/test_topic_spec.rb +39 -0
  86. data/spec/spec_helper.rb +50 -0
  87. data/spec/support/kafka.rb +74 -0
  88. data/spec/support/middleware_tap.rb +12 -0
  89. data/spec/support/skip.rb +9 -0
  90. data/spec/support/test_topic.rb +48 -0
  91. data/spec/unit/nulogy_message_bus_consumer/config_spec.rb +20 -0
  92. data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb +35 -0
  93. data/spec/unit/nulogy_message_bus_consumer/message_spec.rb +84 -0
  94. data/spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb +49 -0
  95. data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb +58 -0
  96. data/spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb +56 -0
  97. data/spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb +70 -0
  98. data/spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb +35 -0
  99. data/spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb +67 -0
  100. data/spec/unit/nulogy_message_bus_consumer_spec.rb +30 -0
  101. metadata +209 -21
  102. data/lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb +0 -51
@@ -0,0 +1,49 @@
1
+ RSpec.describe NulogyMessageBusConsumer::Pipeline do
2
+ let(:handler) { spy }
3
+
4
+ describe "#invoke_pipeline" do
5
+ it "raises if handler is not provided" do
6
+ expect { invoke_pipeline }.to raise_error("Handlers are the end of the line. Do not use yield.")
7
+ end
8
+
9
+ it "calls the handler with message" do
10
+ message = NulogyMessageBusConsumer::Message.new
11
+
12
+ expect(handler).to receive(:call).with(message: message)
13
+
14
+ invoke_pipeline(
15
+ ->(**_, &block) { block.yield(message: message) },
16
+ handler
17
+ )
18
+ end
19
+
20
+ it "passes information along" do
21
+ message = NulogyMessageBusConsumer::Message.new
22
+
23
+ expect(handler).to receive(:call).with(message: message)
24
+
25
+ invoke_pipeline(
26
+ ->(**_, &block) { block.yield(message: message) },
27
+ ->(**_, &block) { block.yield },
28
+ handler
29
+ )
30
+ end
31
+
32
+ it "raises when overwriting an existing key" do
33
+ message = NulogyMessageBusConsumer::Message.new
34
+ other_message = NulogyMessageBusConsumer::Message.new
35
+
36
+ expect {
37
+ invoke_pipeline(
38
+ ->(**_, &block) { block.yield(message: message) },
39
+ ->(**_, &block) { block.yield(message: other_message) },
40
+ handler
41
+ )
42
+ }.to raise_error("Cannot override existing argument(s): message")
43
+ end
44
+ end
45
+
46
+ def invoke_pipeline(*steps)
47
+ NulogyMessageBusConsumer::Pipeline.new(steps).invoke
48
+ end
49
+ end
@@ -0,0 +1,58 @@
1
+ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
2
+ subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new }
3
+
4
+ let(:kafka_consumer) { spy }
5
+ let(:message) { NulogyMessageBusConsumer::Message.new }
6
+
7
+ context "when result is :success" do
8
+ it "returns the result" do
9
+ result = step.call(kafka_consumer: kafka_consumer, message: message) { :success }
10
+
11
+ expect(result).to be(:success)
12
+ end
13
+
14
+ it "commits the message offset" do
15
+ expect(kafka_consumer).to receive(:store_offset).with(message)
16
+ expect(kafka_consumer).to receive(:commit)
17
+
18
+ step.call(kafka_consumer: kafka_consumer, message: message) { :success }
19
+ end
20
+ end
21
+
22
+ context "when result is :failure" do
23
+ it "returns the result" do
24
+ result = step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
25
+
26
+ expect(result).to be(:failure)
27
+ end
28
+
29
+ it "reconnects to Kafka when failure" do
30
+ # When subscribing, we subscribe to a topic (i.e. the keys)
31
+ topic_partition_list = {topic_name: %i[partition_1 partition_2]}
32
+ allow(kafka_consumer).to receive(:subscription).and_return(topic_partition_list)
33
+
34
+ expect(kafka_consumer).to receive(:unsubscribe)
35
+ expect(kafka_consumer).to receive(:subscribe).with(:topic_name)
36
+
37
+ step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
38
+ end
39
+ end
40
+
41
+ context "when result is something else" do
42
+ it "raises" do
43
+ expect {
44
+ step.call(kafka_consumer: kafka_consumer, message: message) { :bogus_value }
45
+ }.to raise_error("'bogus_value' is not a valid processing outcome. Must be :success or :failure")
46
+ end
47
+ end
48
+
49
+ context "when downstream steps raise" do
50
+ it "does not commit" do
51
+ expect(kafka_consumer).not_to receive(:commit)
52
+
53
+ expect {
54
+ step.call(kafka_consumer: kafka_consumer, message: message) { raise "oopsie" }
55
+ }.to raise_error("oopsie")
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,56 @@
1
+ RSpec.describe NulogyMessageBusConsumer::Steps::DeduplicateMessages do
2
+ subject(:deduplicate_messages) { NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger) }
3
+
4
+ let(:logger) { spy }
5
+ let(:id) { SecureRandom.uuid }
6
+ let(:message) { NulogyMessageBusConsumer::Message.new(id: id) }
7
+
8
+ context "when a duplicate message is consumed" do
9
+ it "logs a duplicate was detected" do
10
+ expect(logger).to receive(:warn).with(include_json(
11
+ event: "duplicate_message_detected",
12
+ kafka_message_id: id
13
+ ))
14
+
15
+ deduplicate_messages.call(message: message) { :success }
16
+ deduplicate_messages.call(message: message) { raise "Not evaluated" }
17
+ end
18
+
19
+ it "returns :success" do
20
+ deduplicate_messages.call(message: message) { :success }
21
+
22
+ result = deduplicate_messages.call(message: message) { raise "Not evaluated" }
23
+
24
+ expect(result).to be(:success)
25
+ end
26
+ end
27
+
28
+ context "when a duplicate message is processed at the same time" do
29
+ it "logs a duplicate was detected" do
30
+ expect(logger).to receive(:warn).with(include_json(
31
+ event: "duplicate_message_detected",
32
+ kafka_message_id: id
33
+ ))
34
+
35
+ deduplicate_messages.call(message: message) do
36
+ NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
37
+ :success
38
+ end
39
+ end
40
+
41
+ it "returns :success" do
42
+ result = deduplicate_messages.call(message: message) {
43
+ NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
44
+ :success
45
+ }
46
+
47
+ expect(result).to be(:success)
48
+ end
49
+ end
50
+
51
+ it "records message was processed" do
52
+ deduplicate_messages.call(message: message) { :success }
53
+
54
+ expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(id: id)).to be(true)
55
+ end
56
+ end
@@ -0,0 +1,70 @@
1
+ RSpec.describe NulogyMessageBusConsumer::Steps::LogMessages do
2
+ let(:logger) { spy }
3
+
4
+ it "logs the message was received" do
5
+ log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
6
+
7
+ expect(logger).to receive(:info).with(include_json(
8
+ event: "message_received",
9
+ kafka_message_id: "id",
10
+ message: "Received id"
11
+ ))
12
+
13
+ log_message.call(message: message) {}
14
+ end
15
+
16
+ it "logs the message was processed" do
17
+ log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
18
+
19
+ expect(logger).to receive(:info).with(include_json(
20
+ event: "message_processed",
21
+ kafka_message_id: "id",
22
+ message: match(/^Processed id/)
23
+ ))
24
+
25
+ log_message.call(message: message) {}
26
+ end
27
+
28
+ it "logs the message's time to processed" do
29
+ # clock always gives milliseconds since epoch
30
+ clock = instance_double(NulogyMessageBusConsumer::Clock, ms: 1_000_001_234)
31
+ log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger, clock: clock)
32
+
33
+ expect(logger).to receive(:info).with(include_json(
34
+ event: "message_processed",
35
+ kafka_message_id: "id",
36
+ time_to_processed: 1234
37
+ ))
38
+
39
+ # debezium converts time with zone to nanoseconds since epoch
40
+ log_message.call(message: message(created_at: 1_000_000_000_000)) {}
41
+ end
42
+
43
+ it "logs the message's result" do
44
+ log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
45
+
46
+ expect(logger).to receive(:info).with(include_json(
47
+ event: "message_processed",
48
+ result: "success"
49
+ ))
50
+
51
+ log_message.call(message: message) { :success }
52
+
53
+ expect(logger).to receive(:info).with(include_json(
54
+ event: "message_processed",
55
+ result: "failure"
56
+ ))
57
+
58
+ log_message.call(message: message) { :failure }
59
+ end
60
+
61
+ it "returns a result" do
62
+ log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
63
+
64
+ expect(log_message.call(message: message) { :return_value }).to eq(:return_value)
65
+ end
66
+
67
+ def message(id: "id", created_at: 1_000, **attrs)
68
+ NulogyMessageBusConsumer::Message.new(id: id, created_at: created_at, **attrs)
69
+ end
70
+ end
@@ -0,0 +1,35 @@
1
+ RSpec.describe NulogyMessageBusConsumer::Steps::StreamMessages do
2
+ subject(:step) { NulogyMessageBusConsumer::Steps::StreamMessages.new(logger) }
3
+
4
+ let(:config) { NulogyMessageBusConsumer::Config.new }
5
+ let(:logger) { spy }
6
+
7
+ it "logs errors while streaming from kafka" do
8
+ kafka_consumer = instance_double("Rdkafka::Consumer")
9
+ allow(kafka_consumer).to receive(:subscribe)
10
+ allow(kafka_consumer).to receive(:unsubscribe)
11
+ allow(kafka_consumer).to receive(:each).and_raise(StandardError, "streaming failed")
12
+
13
+ expect(logger).to receive(:error).with(include_json(
14
+ event: "message_processing_errored",
15
+ class: "StandardError",
16
+ message: "streaming failed"
17
+ ))
18
+
19
+ expect {
20
+ step.call(kafka_consumer: kafka_consumer)
21
+ }.to raise_error(StandardError)
22
+ end
23
+
24
+ context "when handler returns failure" do
25
+ it "does not commit the message in kafka" do
26
+ kafka_consumer = spy
27
+ allow(kafka_consumer).to receive(:each).and_yield(anything)
28
+ allow(NulogyMessageBusConsumer::Message).to receive(:from_kafka).and_return(NulogyMessageBusConsumer::Message.new)
29
+
30
+ expect(kafka_consumer).not_to receive(:commit)
31
+
32
+ step.call(kafka_consumer: kafka_consumer) { :failure }
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,67 @@
1
+ module NulogyMessageBusConsumer
2
+ module Tasks
3
+ RSpec.describe LogConsumerLag::Calculator do
4
+ subject(:calculator) { LogConsumerLag::Calculator }
5
+
6
+ describe ".add_max_lag" do
7
+ it "adds max lag per partition" do
8
+ lag_per_topic = {
9
+ topic_1: {
10
+ partition_1: 100,
11
+ partition_2: 900,
12
+ partition_3: 100
13
+ },
14
+ topic_2: {
15
+ partition_1: 100,
16
+ partition_2: 200,
17
+ partition_3: 100
18
+ }
19
+ }
20
+
21
+ lag_with_max = calculator.add_max_lag(lag_per_topic)
22
+
23
+ expect(lag_with_max).to match(
24
+ _max: 900,
25
+ topic_1: {
26
+ _max: 900,
27
+ partition_1: 100,
28
+ partition_2: 900,
29
+ partition_3: 100
30
+ },
31
+ topic_2: {
32
+ _max: 200,
33
+ partition_1: 100,
34
+ partition_2: 200,
35
+ partition_3: 100
36
+ }
37
+ )
38
+ end
39
+
40
+ it "sets the max to 0 when no partitions are subscribed to" do
41
+ lag_per_topic = {
42
+ topic_1: {}
43
+ }
44
+
45
+ lag_with_max = calculator.add_max_lag(lag_per_topic)
46
+
47
+ expect(lag_with_max).to match(
48
+ _max: 0,
49
+ topic_1: {
50
+ _max: 0
51
+ }
52
+ )
53
+ end
54
+
55
+ it "sets the max to 0 when no topics are subscribed to" do
56
+ lag_per_topic = {}
57
+
58
+ lag_with_max = calculator.add_max_lag(lag_per_topic)
59
+
60
+ expect(lag_with_max).to match(
61
+ _max: 0
62
+ )
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,30 @@
1
+ RSpec.describe NulogyMessageBusConsumer do
2
+ describe "#configure" do
3
+ before do
4
+ config = NulogyMessageBusConsumer.config
5
+ config.instance_variable_names.each do |variable_name|
6
+ config.remove_instance_variable(variable_name)
7
+ end
8
+ end
9
+
10
+ it "allows configuration to be provided as arguments" do
11
+ NulogyMessageBusConsumer.configure(
12
+ bootstrap_servers: "some servers"
13
+ )
14
+
15
+ expect(NulogyMessageBusConsumer.config).to have_attributes(
16
+ bootstrap_servers: "some servers"
17
+ )
18
+ end
19
+
20
+ it "allows configuration to be provided as a block" do
21
+ NulogyMessageBusConsumer.configure do |config|
22
+ config.bootstrap_servers = "some servers"
23
+ end
24
+
25
+ expect(NulogyMessageBusConsumer.config).to have_attributes(
26
+ bootstrap_servers: "some servers"
27
+ )
28
+ end
29
+ end
30
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nulogy_message_bus_consumer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nulogy
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-12 00:00:00.000000000 Z
11
+ date: 2021-04-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -52,6 +52,34 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler-audit
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.7.0.1
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.7.0.1
69
+ - !ruby/object:Gem::Dependency
70
+ name: dotenv
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '='
74
+ - !ruby/object:Gem::Version
75
+ version: 2.7.6
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '='
81
+ - !ruby/object:Gem::Version
82
+ version: 2.7.6
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: pg
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -100,42 +128,42 @@ dependencies:
100
128
  requirements:
101
129
  - - '='
102
130
  - !ruby/object:Gem::Version
103
- version: 6.0.3
131
+ version: 6.0.3.5
104
132
  type: :development
105
133
  prerelease: false
106
134
  version_requirements: !ruby/object:Gem::Requirement
107
135
  requirements:
108
136
  - - '='
109
137
  - !ruby/object:Gem::Version
110
- version: 6.0.3
138
+ version: 6.0.3.5
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: rake-release
113
141
  requirement: !ruby/object:Gem::Requirement
114
142
  requirements:
115
143
  - - '='
116
144
  - !ruby/object:Gem::Version
117
- version: 1.2.1
145
+ version: 1.3.0
118
146
  type: :development
119
147
  prerelease: false
120
148
  version_requirements: !ruby/object:Gem::Requirement
121
149
  requirements:
122
150
  - - '='
123
151
  - !ruby/object:Gem::Version
124
- version: 1.2.1
152
+ version: 1.3.0
125
153
  - !ruby/object:Gem::Dependency
126
154
  name: rspec
127
155
  requirement: !ruby/object:Gem::Requirement
128
156
  requirements:
129
- - - '='
157
+ - - "~>"
130
158
  - !ruby/object:Gem::Version
131
- version: 3.9.0
159
+ version: '3.10'
132
160
  type: :development
133
161
  prerelease: false
134
162
  version_requirements: !ruby/object:Gem::Requirement
135
163
  requirements:
136
- - - '='
164
+ - - "~>"
137
165
  - !ruby/object:Gem::Version
138
- version: 3.9.0
166
+ version: '3.10'
139
167
  - !ruby/object:Gem::Dependency
140
168
  name: rspec-json_expectations
141
169
  requirement: !ruby/object:Gem::Requirement
@@ -165,19 +193,19 @@ dependencies:
165
193
  - !ruby/object:Gem::Version
166
194
  version: 4.0.1
167
195
  - !ruby/object:Gem::Dependency
168
- name: rubocop
196
+ name: standard
169
197
  requirement: !ruby/object:Gem::Requirement
170
198
  requirements:
171
199
  - - '='
172
200
  - !ruby/object:Gem::Version
173
- version: 0.81.0
201
+ version: 0.11.0
174
202
  type: :development
175
203
  prerelease: false
176
204
  version_requirements: !ruby/object:Gem::Requirement
177
205
  requirements:
178
206
  - - '='
179
207
  - !ruby/object:Gem::Version
180
- version: 0.81.0
208
+ version: 0.11.0
181
209
  - !ruby/object:Gem::Dependency
182
210
  name: rubocop-rails
183
211
  requirement: !ruby/object:Gem::Requirement
@@ -206,7 +234,7 @@ dependencies:
206
234
  - - '='
207
235
  - !ruby/object:Gem::Version
208
236
  version: 1.38.1
209
- description:
237
+ description:
210
238
  email:
211
239
  - tass@nulogy.com
212
240
  executables: []
@@ -214,13 +242,18 @@ extensions: []
214
242
  extra_rdoc_files: []
215
243
  files:
216
244
  - Rakefile
245
+ - config/credentials/message-bus-us-east-1.key
246
+ - config/credentials/message-bus-us-east-1.yml.enc
217
247
  - config/routes.rb
218
248
  - db/migrate/20200509095105_create_message_bus_processed_messages.rb
219
249
  - lib/nulogy_message_bus_consumer.rb
250
+ - lib/nulogy_message_bus_consumer/clock.rb
220
251
  - lib/nulogy_message_bus_consumer/config.rb
252
+ - lib/nulogy_message_bus_consumer/deployment/ecs.rb
221
253
  - lib/nulogy_message_bus_consumer/engine.rb
222
254
  - lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb
223
255
  - lib/nulogy_message_bus_consumer/kafka_utils.rb
256
+ - lib/nulogy_message_bus_consumer/lag_tracker.rb
224
257
  - lib/nulogy_message_bus_consumer/message.rb
225
258
  - lib/nulogy_message_bus_consumer/null_logger.rb
226
259
  - lib/nulogy_message_bus_consumer/pipeline.rb
@@ -229,17 +262,96 @@ files:
229
262
  - lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
230
263
  - lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
231
264
  - lib/nulogy_message_bus_consumer/steps/log_messages.rb
232
- - lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb
233
265
  - lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb
234
266
  - lib/nulogy_message_bus_consumer/steps/stream_messages.rb
235
267
  - lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb
268
+ - lib/nulogy_message_bus_consumer/steps/timed_task.rb
269
+ - lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb
270
+ - lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb
271
+ - lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb
236
272
  - lib/nulogy_message_bus_consumer/version.rb
237
273
  - lib/tasks/engine/message_bus_consumer.rake
274
+ - spec/dummy/Rakefile
275
+ - spec/dummy/app/assets/config/manifest.js
276
+ - spec/dummy/app/assets/stylesheets/application.css
277
+ - spec/dummy/app/channels/application_cable/channel.rb
278
+ - spec/dummy/app/channels/application_cable/connection.rb
279
+ - spec/dummy/app/controllers/application_controller.rb
280
+ - spec/dummy/app/helpers/application_helper.rb
281
+ - spec/dummy/app/javascript/packs/application.js
282
+ - spec/dummy/app/jobs/application_job.rb
283
+ - spec/dummy/app/mailers/application_mailer.rb
284
+ - spec/dummy/app/models/application_record.rb
285
+ - spec/dummy/app/views/layouts/application.html.erb
286
+ - spec/dummy/app/views/layouts/mailer.html.erb
287
+ - spec/dummy/app/views/layouts/mailer.text.erb
288
+ - spec/dummy/bin/rails
289
+ - spec/dummy/bin/rake
290
+ - spec/dummy/bin/setup
291
+ - spec/dummy/config.ru
292
+ - spec/dummy/config/application.rb
293
+ - spec/dummy/config/boot.rb
294
+ - spec/dummy/config/cable.yml
295
+ - spec/dummy/config/credentials/message-bus-us-east-1.key
296
+ - spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
297
+ - spec/dummy/config/database.yml
298
+ - spec/dummy/config/environment.rb
299
+ - spec/dummy/config/environments/development.rb
300
+ - spec/dummy/config/environments/production.rb
301
+ - spec/dummy/config/environments/test.rb
302
+ - spec/dummy/config/initializers/application_controller_renderer.rb
303
+ - spec/dummy/config/initializers/assets.rb
304
+ - spec/dummy/config/initializers/backtrace_silencers.rb
305
+ - spec/dummy/config/initializers/content_security_policy.rb
306
+ - spec/dummy/config/initializers/cookies_serializer.rb
307
+ - spec/dummy/config/initializers/filter_parameter_logging.rb
308
+ - spec/dummy/config/initializers/inflections.rb
309
+ - spec/dummy/config/initializers/message_bus_consumer.rb
310
+ - spec/dummy/config/initializers/mime_types.rb
311
+ - spec/dummy/config/initializers/wrap_parameters.rb
312
+ - spec/dummy/config/locales/en.yml
313
+ - spec/dummy/config/puma.rb
314
+ - spec/dummy/config/routes.rb
315
+ - spec/dummy/config/spring.rb
316
+ - spec/dummy/config/storage.yml
317
+ - spec/dummy/db/schema.rb
318
+ - spec/dummy/log/development.log
319
+ - spec/dummy/log/production.log
320
+ - spec/dummy/log/test.log
321
+ - spec/dummy/public/404.html
322
+ - spec/dummy/public/422.html
323
+ - spec/dummy/public/500.html
324
+ - spec/dummy/public/apple-touch-icon-precomposed.png
325
+ - spec/dummy/public/apple-touch-icon.png
326
+ - spec/dummy/public/favicon.ico
327
+ - spec/dummy/tmp/development_secret.txt
328
+ - spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
329
+ - spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
330
+ - spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
331
+ - spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
332
+ - spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
333
+ - spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
334
+ - spec/integration/test_topic_spec.rb
335
+ - spec/spec_helper.rb
336
+ - spec/support/kafka.rb
337
+ - spec/support/middleware_tap.rb
338
+ - spec/support/skip.rb
339
+ - spec/support/test_topic.rb
340
+ - spec/unit/nulogy_message_bus_consumer/config_spec.rb
341
+ - spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
342
+ - spec/unit/nulogy_message_bus_consumer/message_spec.rb
343
+ - spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
344
+ - spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
345
+ - spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
346
+ - spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
347
+ - spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
348
+ - spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
349
+ - spec/unit/nulogy_message_bus_consumer_spec.rb
238
350
  homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
239
351
  licenses: []
240
352
  metadata:
241
353
  allowed_push_host: https://rubygems.org/
242
- post_install_message:
354
+ post_install_message:
243
355
  rdoc_options: []
244
356
  require_paths:
245
357
  - lib
@@ -247,15 +359,91 @@ required_ruby_version: !ruby/object:Gem::Requirement
247
359
  requirements:
248
360
  - - ">="
249
361
  - !ruby/object:Gem::Version
250
- version: '0'
362
+ version: '2.6'
251
363
  required_rubygems_version: !ruby/object:Gem::Requirement
252
364
  requirements:
253
365
  - - ">="
254
366
  - !ruby/object:Gem::Version
255
367
  version: '0'
256
368
  requirements: []
257
- rubygems_version: 3.0.3
258
- signing_key:
369
+ rubygems_version: 3.2.15
370
+ signing_key:
259
371
  specification_version: 4
260
372
  summary: Code for accessing the Nulogy Message Bus
261
- test_files: []
373
+ test_files:
374
+ - spec/dummy/Rakefile
375
+ - spec/dummy/app/assets/config/manifest.js
376
+ - spec/dummy/app/assets/stylesheets/application.css
377
+ - spec/dummy/app/channels/application_cable/channel.rb
378
+ - spec/dummy/app/channels/application_cable/connection.rb
379
+ - spec/dummy/app/controllers/application_controller.rb
380
+ - spec/dummy/app/helpers/application_helper.rb
381
+ - spec/dummy/app/javascript/packs/application.js
382
+ - spec/dummy/app/jobs/application_job.rb
383
+ - spec/dummy/app/mailers/application_mailer.rb
384
+ - spec/dummy/app/models/application_record.rb
385
+ - spec/dummy/app/views/layouts/application.html.erb
386
+ - spec/dummy/app/views/layouts/mailer.html.erb
387
+ - spec/dummy/app/views/layouts/mailer.text.erb
388
+ - spec/dummy/bin/rails
389
+ - spec/dummy/bin/rake
390
+ - spec/dummy/bin/setup
391
+ - spec/dummy/config/application.rb
392
+ - spec/dummy/config/boot.rb
393
+ - spec/dummy/config/cable.yml
394
+ - spec/dummy/config/credentials/message-bus-us-east-1.key
395
+ - spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
396
+ - spec/dummy/config/database.yml
397
+ - spec/dummy/config/environment.rb
398
+ - spec/dummy/config/environments/development.rb
399
+ - spec/dummy/config/environments/production.rb
400
+ - spec/dummy/config/environments/test.rb
401
+ - spec/dummy/config/initializers/application_controller_renderer.rb
402
+ - spec/dummy/config/initializers/assets.rb
403
+ - spec/dummy/config/initializers/backtrace_silencers.rb
404
+ - spec/dummy/config/initializers/content_security_policy.rb
405
+ - spec/dummy/config/initializers/cookies_serializer.rb
406
+ - spec/dummy/config/initializers/filter_parameter_logging.rb
407
+ - spec/dummy/config/initializers/inflections.rb
408
+ - spec/dummy/config/initializers/message_bus_consumer.rb
409
+ - spec/dummy/config/initializers/mime_types.rb
410
+ - spec/dummy/config/initializers/wrap_parameters.rb
411
+ - spec/dummy/config/locales/en.yml
412
+ - spec/dummy/config/puma.rb
413
+ - spec/dummy/config/routes.rb
414
+ - spec/dummy/config/spring.rb
415
+ - spec/dummy/config/storage.yml
416
+ - spec/dummy/config.ru
417
+ - spec/dummy/db/schema.rb
418
+ - spec/dummy/log/development.log
419
+ - spec/dummy/log/production.log
420
+ - spec/dummy/log/test.log
421
+ - spec/dummy/public/404.html
422
+ - spec/dummy/public/422.html
423
+ - spec/dummy/public/500.html
424
+ - spec/dummy/public/apple-touch-icon-precomposed.png
425
+ - spec/dummy/public/apple-touch-icon.png
426
+ - spec/dummy/public/favicon.ico
427
+ - spec/dummy/tmp/development_secret.txt
428
+ - spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
429
+ - spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
430
+ - spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
431
+ - spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
432
+ - spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
433
+ - spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
434
+ - spec/integration/test_topic_spec.rb
435
+ - spec/spec_helper.rb
436
+ - spec/support/kafka.rb
437
+ - spec/support/middleware_tap.rb
438
+ - spec/support/skip.rb
439
+ - spec/support/test_topic.rb
440
+ - spec/unit/nulogy_message_bus_consumer/config_spec.rb
441
+ - spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
442
+ - spec/unit/nulogy_message_bus_consumer/message_spec.rb
443
+ - spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
444
+ - spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
445
+ - spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
446
+ - spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
447
+ - spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
448
+ - spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
449
+ - spec/unit/nulogy_message_bus_consumer_spec.rb