nulogy_message_bus_producer 3.2.0 → 3.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -1
- data/lib/nulogy_message_bus_producer.rb +3 -6
- data/lib/nulogy_message_bus_producer/repopulate_replication_slots.rb +7 -5
- data/lib/nulogy_message_bus_producer/subscriptions/no_variables.rb +43 -0
- data/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker.rb +17 -5
- data/lib/nulogy_message_bus_producer/version.rb +1 -1
- data/lib/tasks/engine/message_bus_producer.rake +4 -4
- data/spec/dummy/log/development.log +49 -0
- data/spec/dummy/log/test.log +9492 -0
- data/spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb +16 -8
- data/spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb +2 -2
- data/spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb +3 -5
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/no_variables_spec.rb +46 -0
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb +1 -3
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb +1 -3
- data/spec/support/kafka.rb +13 -6
- data/spec/support/spec_utils.rb +3 -2
- data/spec/support/sql_helpers.rb +8 -10
- data/spec/support/test_graphql_schema.rb +1 -0
- metadata +17 -14
@@ -1,5 +1,5 @@
|
|
1
1
|
require "integration_spec_helper"
|
2
|
-
require
|
2
|
+
require "net/http"
|
3
3
|
|
4
4
|
RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
|
5
5
|
let(:company_uuid) { SecureRandom.uuid }
|
@@ -23,20 +23,24 @@ RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
|
|
23
23
|
|
24
24
|
number_of_messages.times { |n| create_event(uuid(n)) }
|
25
25
|
|
26
|
-
|
26
|
+
start_debezium
|
27
27
|
|
28
|
-
|
28
|
+
described_class.repopulate
|
29
29
|
end
|
30
30
|
|
31
31
|
message_payloads = Kafka.wait_for_messages(consumer).map(&:payload)
|
32
|
-
matcher = number_of_messages.times.map { |n| include(uuid(n)) }
|
33
32
|
expect(message_payloads.count).to eq(number_of_messages)
|
33
|
+
matcher = Array.new(number_of_messages) { |n| include(uuid(n)) }
|
34
34
|
expect(message_payloads).to match(matcher)
|
35
35
|
end
|
36
36
|
|
37
37
|
def cleanup_everything
|
38
38
|
truncate_db
|
39
|
-
|
39
|
+
begin
|
40
|
+
Kafka.delete_topic(topic_name)
|
41
|
+
rescue StandardError
|
42
|
+
nil
|
43
|
+
end
|
40
44
|
kafka_connect.delete
|
41
45
|
wait_for_replication_slot_cleanup(replication_slot_name)
|
42
46
|
end
|
@@ -51,7 +55,7 @@ RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
|
|
51
55
|
trigger_event("testCreated", root_object)
|
52
56
|
end
|
53
57
|
|
54
|
-
def
|
58
|
+
def start_debezium # rubocop:disable Metrics/AbcSize
|
55
59
|
config = build_debezium_config
|
56
60
|
|
57
61
|
response = kafka_connect.configure(config)
|
@@ -78,7 +82,7 @@ RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
|
|
78
82
|
end
|
79
83
|
end
|
80
84
|
|
81
|
-
def build_debezium_config
|
85
|
+
def build_debezium_config # rubocop:disable Metrics/MethodLength
|
82
86
|
db_config = Rails.configuration.database_configuration[Rails.env]
|
83
87
|
events_table = NulogyMessageBusProducer::SubscriptionEvent.table_name
|
84
88
|
|
@@ -95,7 +99,11 @@ RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
|
|
95
99
|
|
96
100
|
"behavior.on.null.values": "delete",
|
97
101
|
"connector.class": "io.debezium.connector.postgresql.PostgresConnector",
|
98
|
-
"database.initial.statements":
|
102
|
+
"database.initial.statements": <<~SQL,
|
103
|
+
DO $$BEGIN
|
104
|
+
IF NOT EXISTS(SELECT FROM pg_publication WHERE pubname = 'debezium_public_events')
|
105
|
+
THEN CREATE PUBLICATION debezium_public_events FOR TABLE #{events_table} WITH (publish = 'insert');; END IF;; END$$;
|
106
|
+
SQL
|
99
107
|
"errors.log.enable": "true",
|
100
108
|
"errors.log.include.messages": "true",
|
101
109
|
"heartbeat.interval.ms": "30000",
|
data/spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require "integration_spec_helper"
|
2
2
|
|
3
3
|
RSpec.describe NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator do
|
4
|
-
subject(:validator) {
|
4
|
+
subject(:validator) { described_class.new }
|
5
5
|
|
6
6
|
describe "#validate" do
|
7
7
|
context "when a valid query is present" do
|
@@ -24,7 +24,7 @@ RSpec.describe NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator do
|
|
24
24
|
}
|
25
25
|
GRAPHQL
|
26
26
|
|
27
|
-
subscription.query.gsub!(/\bid\b/,
|
27
|
+
subscription.query.gsub!(/\bid\b/, "a_field_that_does_not_exist")
|
28
28
|
subscription.save(validate: false)
|
29
29
|
subscription
|
30
30
|
end
|
@@ -28,13 +28,12 @@ RSpec.describe NulogyMessageBusProducer::Subscription do
|
|
28
28
|
|
29
29
|
it "is invalid with an invalid query" do
|
30
30
|
model = build_subscription(
|
31
|
-
|
32
|
-
query: subscription_query(query: " foo { a_field_that_does_not_exist }")
|
31
|
+
query: subscription_query(query: "foo { a_field_that_does_not_exist }")
|
33
32
|
)
|
34
33
|
|
35
34
|
model.validate
|
36
35
|
|
37
|
-
expect(model).
|
36
|
+
expect(model).not_to be_valid
|
38
37
|
expect(model.errors[:query]).to contain_exactly(
|
39
38
|
"Field 'a_field_that_does_not_exist' doesn't exist on type 'testObject' (id: <new_record>)"
|
40
39
|
)
|
@@ -42,13 +41,12 @@ RSpec.describe NulogyMessageBusProducer::Subscription do
|
|
42
41
|
|
43
42
|
it "valid with a valid query" do
|
44
43
|
model = build_subscription(
|
45
|
-
schema_key: "test",
|
46
44
|
query: subscription_query(query: "foo { id }")
|
47
45
|
)
|
48
46
|
|
49
47
|
model.validate
|
50
48
|
|
51
|
-
expect(model.errors).
|
49
|
+
expect(model.errors).not_to include(:query)
|
52
50
|
end
|
53
51
|
end
|
54
52
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "integration_spec_helper"
|
2
|
+
|
3
|
+
RSpec.describe NulogyMessageBusProducer::Subscriptions::NoVariables do
|
4
|
+
it "blocks subscriptions with variables" do
|
5
|
+
query = <<~SUBSCRIPTION
|
6
|
+
subscription($sub_id: ID!, $sg_id: ID!, $topic: String!) {
|
7
|
+
testCreated(subscriptionId: $sub_id, subscriptionGroupId: $sg_id, topicName: $topic) {
|
8
|
+
foo { id }
|
9
|
+
}
|
10
|
+
}
|
11
|
+
SUBSCRIPTION
|
12
|
+
|
13
|
+
result = execute_graphql(
|
14
|
+
query,
|
15
|
+
NulogyMessageBusProducer::Specs::TestSchema,
|
16
|
+
variables: {
|
17
|
+
sub_id: SecureRandom.uuid,
|
18
|
+
sg_id: SecureRandom.uuid,
|
19
|
+
topic: "test_topic"
|
20
|
+
}
|
21
|
+
)
|
22
|
+
|
23
|
+
expect(result).to include_json(
|
24
|
+
errors: [{ message: include("Subscriptions should not be created with arguments") }]
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "does not block subscriptions without variables" do
|
29
|
+
query = <<~SUBSCRIPTION
|
30
|
+
subscription {
|
31
|
+
testCreated(
|
32
|
+
subscriptionId: "#{SecureRandom.uuid}",
|
33
|
+
subscriptionGroupId: "#{SecureRandom.uuid}",
|
34
|
+
topicName: "test_topic"
|
35
|
+
) { foo { id } }
|
36
|
+
}
|
37
|
+
SUBSCRIPTION
|
38
|
+
|
39
|
+
result = execute_graphql(
|
40
|
+
query,
|
41
|
+
NulogyMessageBusProducer::Specs::TestSchema
|
42
|
+
)
|
43
|
+
|
44
|
+
expect(result).not_to include(:errors)
|
45
|
+
end
|
46
|
+
end
|
data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb
CHANGED
@@ -46,9 +46,7 @@ RSpec.describe NulogyMessageBusProducer::Subscriptions::PostgresTransport do
|
|
46
46
|
|
47
47
|
event = NulogyMessageBusProducer::SubscriptionEvent.find_by!(subscription_id: subscription.id)
|
48
48
|
expect(event).to have_attributes(
|
49
|
-
event_json: include_json(
|
50
|
-
testCreated: { foo: { contextData: "some contextual information" } }
|
51
|
-
)
|
49
|
+
event_json: include_json(testCreated: { foo: { contextData: "some contextual information" } })
|
52
50
|
)
|
53
51
|
end
|
54
52
|
|
@@ -34,7 +34,7 @@ RSpec.describe NulogyMessageBusProducer::Subscriptions::RiskySubscriptionBlocker
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def attempt_subscription(query)
|
37
|
-
|
37
|
+
execute_graphql(<<~GRAPHQL, NulogyMessageBusProducer::Specs::TestSchema)
|
38
38
|
subscription {
|
39
39
|
testCreated (
|
40
40
|
subscriptionId: "#{SecureRandom.uuid}",
|
@@ -45,7 +45,5 @@ RSpec.describe NulogyMessageBusProducer::Subscriptions::RiskySubscriptionBlocker
|
|
45
45
|
}
|
46
46
|
}
|
47
47
|
GRAPHQL
|
48
|
-
|
49
|
-
execute_graphql(gql, NulogyMessageBusProducer::Specs::TestSchema)
|
50
48
|
end
|
51
49
|
end
|
data/spec/support/kafka.rb
CHANGED
@@ -44,11 +44,9 @@ module Kafka
|
|
44
44
|
loop do
|
45
45
|
message = consumer.poll(timeout)
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
return messages
|
51
|
-
end
|
47
|
+
return messages unless message
|
48
|
+
|
49
|
+
messages << message
|
52
50
|
end
|
53
51
|
end
|
54
52
|
|
@@ -62,7 +60,16 @@ module Kafka
|
|
62
60
|
end
|
63
61
|
|
64
62
|
def create_topic(topic_name)
|
65
|
-
|
63
|
+
cmd = [
|
64
|
+
"docker-compose exec",
|
65
|
+
"-T kafka kafka-topics",
|
66
|
+
"--zookeeper zookeeper:2181",
|
67
|
+
"--create",
|
68
|
+
"--replication-factor 1",
|
69
|
+
"--partitions 3",
|
70
|
+
"--if-not-exists --topic #{topic_name}"
|
71
|
+
]
|
72
|
+
run(cmd.join(" "))
|
66
73
|
end
|
67
74
|
|
68
75
|
def delete_topic(topic_name)
|
data/spec/support/spec_utils.rb
CHANGED
@@ -4,12 +4,13 @@ module SpecUtils
|
|
4
4
|
def wait_for(attempts: 100, interval: 0.1)
|
5
5
|
attempts.times do
|
6
6
|
return if yield
|
7
|
+
|
7
8
|
sleep interval
|
8
9
|
end
|
9
10
|
raise "Waited for #{attempts} times but it never resolved"
|
10
11
|
end
|
11
12
|
|
12
13
|
def uuid(entity_number)
|
13
|
-
|
14
|
+
format("00000000-0000-0000-0000-%<id>12.12d", id: entity_number)
|
14
15
|
end
|
15
|
-
end
|
16
|
+
end
|
data/spec/support/sql_helpers.rb
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
module SqlHelpers
|
2
2
|
def without_transaction
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
truncate_db
|
8
|
-
end
|
3
|
+
ActiveRecord::Base.connection.rollback_transaction
|
4
|
+
yield
|
5
|
+
ensure
|
6
|
+
truncate_db
|
9
7
|
end
|
10
8
|
|
11
9
|
def truncate_db
|
@@ -19,7 +17,7 @@ module SqlHelpers
|
|
19
17
|
ActiveRecord::Base.connection.execute("TRUNCATE #{NulogyMessageBusProducer::Subscription.table_name}")
|
20
18
|
end
|
21
19
|
|
22
|
-
def
|
20
|
+
def replication_slots
|
23
21
|
results = ActiveRecord::Base.connection.exec_query(<<~SQL)
|
24
22
|
SELECT slot_name FROM pg_replication_slots
|
25
23
|
SQL
|
@@ -35,13 +33,13 @@ module SqlHelpers
|
|
35
33
|
|
36
34
|
def wait_for_replication_slot(slot_name)
|
37
35
|
wait_for do
|
38
|
-
|
36
|
+
replication_slots.any? { |replication_slot| replication_slot == slot_name }
|
39
37
|
end
|
40
38
|
end
|
41
39
|
|
42
40
|
def wait_for_replication_slot_cleanup(slot_name)
|
43
41
|
wait_for do
|
44
|
-
|
42
|
+
replication_slots.none? { |replication_slot| replication_slot == slot_name }
|
45
43
|
end
|
46
44
|
end
|
47
|
-
end
|
45
|
+
end
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nulogy_message_bus_producer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.2.
|
4
|
+
version: 3.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nulogy
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: graphql
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.11.4
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 1.11.4
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rails
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: '5'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: '5'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: pg
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -226,6 +226,7 @@ files:
|
|
226
226
|
- lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator.rb
|
227
227
|
- lib/nulogy_message_bus_producer/subscription.rb
|
228
228
|
- lib/nulogy_message_bus_producer/subscription_event.rb
|
229
|
+
- lib/nulogy_message_bus_producer/subscriptions/no_variables.rb
|
229
230
|
- lib/nulogy_message_bus_producer/subscriptions/postgres_transport.rb
|
230
231
|
- lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker.rb
|
231
232
|
- lib/nulogy_message_bus_producer/version.rb
|
@@ -288,6 +289,7 @@ files:
|
|
288
289
|
- spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb
|
289
290
|
- spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb
|
290
291
|
- spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb
|
292
|
+
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/no_variables_spec.rb
|
291
293
|
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb
|
292
294
|
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb
|
293
295
|
- spec/integration_spec_helper.rb
|
@@ -307,9 +309,9 @@ require_paths:
|
|
307
309
|
- lib
|
308
310
|
required_ruby_version: !ruby/object:Gem::Requirement
|
309
311
|
requirements:
|
310
|
-
- - "
|
312
|
+
- - "~>"
|
311
313
|
- !ruby/object:Gem::Version
|
312
|
-
version: '0'
|
314
|
+
version: '2.0'
|
313
315
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
314
316
|
requirements:
|
315
317
|
- - ">="
|
@@ -381,6 +383,7 @@ test_files:
|
|
381
383
|
- spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb
|
382
384
|
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb
|
383
385
|
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb
|
386
|
+
- spec/integration/lib/nulogy_message_bus_producer/subscriptions/no_variables_spec.rb
|
384
387
|
- spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb
|
385
388
|
- spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb
|
386
389
|
- spec/support/kafka_connect.rb
|