nulogy_message_bus_producer 1.0.4 → 3.2.0

data/spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb

@@ -0,0 +1,133 @@
+require "integration_spec_helper"
+require 'net/http'
+
+RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
+  let(:company_uuid) { SecureRandom.uuid }
+  let(:number_of_messages) { 100 }
+
+  let(:kafka_bootstrap_servers) { "host.docker.internal:39092" }
+  let(:kafka_connect) { KafkaConnect.new("http://localhost:8083", "ruby_specs") }
+  let(:replication_slot_name) { "rspec_replication_slot" }
+  let(:topic_name) { "repopulate_replication_slot_tests" }
+
+  before do
+    cleanup_everything
+  end
+
+  it "generates events" do
+    Kafka.create_topic(topic_name)
+    consumer = Kafka.setup_kafka_consumer(topic_name)
+
+    without_transaction do
+      subscribe_to(event_type: "testCreated", topic_name: topic_name)
+
+      number_of_messages.times { |n| create_event(uuid(n)) }
+
+      configure_debezium
+
+      NulogyMessageBusProducer::RepopulateReplicationSlots.repopulate
+    end
+
+    message_payloads = Kafka.wait_for_messages(consumer).map(&:payload)
+    matcher = number_of_messages.times.map { |n| include(uuid(n)) }
+    expect(message_payloads.count).to eq(number_of_messages)
+    expect(message_payloads).to match(matcher)
+  end
+
+  def cleanup_everything
+    truncate_db
+    Kafka.delete_topic(topic_name) rescue nil
+    kafka_connect.delete
+    wait_for_replication_slot_cleanup(replication_slot_name)
+  end
+
+  def create_event(entity_uuid)
+    root_object = {
+      company_uuid: company_uuid,
+      foo: {
+        id: entity_uuid
+      }
+    }
+    trigger_event("testCreated", root_object)
+  end
+
+  def configure_debezium
+    config = build_debezium_config
+
+    response = kafka_connect.configure(config)
+
+    expect(response.code).to eq("201"), <<~MESSAGE
+      Creating a Debezium config in Kafka Connect has failed. HTTP Request returned:
+      Code: #{response.code}
+      #{JSON.parse(response.body).pretty_inspect}
+      #{config.pretty_inspect}
+    MESSAGE
+    wait_for_tasks_to_start(kafka_connect)
+    wait_for_replication_slot(replication_slot_name)
+  end
+
+  def wait_for_tasks_to_start(kafka_connect)
+    SpecUtils.wait_for(attempts: 10, interval: 0.5) do
+      tasks = kafka_connect.status[:tasks]
+      next false if tasks.blank?
+
+      expect(tasks.all? { |task| task[:state] == "RUNNING" }).to eq(true), <<~MESSAGE
+        Expected the Kafka Connect tasks to be running. Instead found:
+        #{tasks.pretty_inspect}
+      MESSAGE
+    end
+  end
+
+  def build_debezium_config
+    db_config = Rails.configuration.database_configuration[Rails.env]
+    events_table = NulogyMessageBusProducer::SubscriptionEvent.table_name
+
+    {
+      "bootstrap.servers": kafka_bootstrap_servers,
+
+      "database.dbname": db_config["database"],
+      "database.hostname": db_config["host"] == "localhost" ? "host.docker.internal" : db_config["host"],
+      "database.password": db_config["password"],
+      "database.port": db_config["port"] || 5432,
+      "database.server.name": "test-environment",
+      "database.user": db_config["username"],
+      "slot.name": replication_slot_name,
+
+      "behavior.on.null.values": "delete",
+      "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+      "database.initial.statements": "DO $$BEGIN IF not exists(select from pg_publication where pubname = 'debezium_public_events') THEN CREATE PUBLICATION debezium_public_events FOR TABLE #{events_table} WITH (publish = 'insert');; END IF;; END$$;",
+      "errors.log.enable": "true",
+      "errors.log.include.messages": "true",
+      "heartbeat.interval.ms": "30000",
+      "plugin.name": "pgoutput",
+      "publication.name": "debezium_public_events",
+      "slot.drop.on.stop": "true",
+      "snapshot.mode": "never",
+      "table.whitelist": "public.#{events_table}",
+
+      "transforms": "requireTopicName,unwrap,extractTopicName,extractPartitionKey,removeFields",
+
+      "transforms.requireTopicName.type": "io.confluent.connect.transforms.Filter$Value",
+      "transforms.requireTopicName.filter.condition": "$.after.topic_name",
+      "transforms.requireTopicName.filter.type": "include",
+      "transforms.requireTopicName.missing.or.null.behavior": "exclude",
+
+      "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
+      "transforms.unwrap.drop.tombstones": "true",
+
+      "transforms.extractTopicName.type": "io.confluent.connect.transforms.ExtractTopic$Value",
+      "transforms.extractTopicName.field": "topic_name",
+
+      "transforms.extractPartitionKey.type": "org.apache.kafka.connect.transforms.ValueToKey",
+      "transforms.extractPartitionKey.fields": "partition_key",
+
+      "transforms.removeFields.type": "org.apache.kafka.connect.transforms.ReplaceField$Value",
+      "transforms.removeFields.blacklist": "topic_name,partition_key",
+
+      "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+      "key.converter.schemas.enable": "false",
+      "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+      "value.converter.schemas.enable": "false"
+    }
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb

@@ -0,0 +1,49 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator do
+  subject(:validator) { NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator.new }
+
+  describe "#validate" do
+    context "when a valid query is present" do
+      it "return true" do
+        subscribe_to(query: <<~GRAPHQL)
+          foo {
+            id
+          }
+        GRAPHQL
+
+        expect(validator.validate).to be(true)
+      end
+    end
+
+    context "when an invalid query is present" do
+      let(:subscription_with_error) do
+        subscription = subscribe_to(query: <<~GRAPHQL)
+          foo {
+            id
+          }
+        GRAPHQL
+
+        subscription.query.gsub!(/\bid\b/, 'a_field_that_does_not_exist')
+        subscription.save(validate: false)
+        subscription
+      end
+
+      it "returns false" do
+        subscription_with_error
+
+        expect(validator.validate).to be(false)
+      end
+
+      it "has errors" do
+        subscription_with_error
+
+        validator.validate
+
+        expect(validator.errors).to contain_exactly(
+          "Field 'a_field_that_does_not_exist' doesn't exist on type 'testObject' (id: #{subscription_with_error.id})"
+        )
+      end
+    end
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb

@@ -0,0 +1,63 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscription do
+  context "when validating" do
+    it "is invalid with a blank query" do
+      model = build_subscription(query: "")
+
+      model.validate
+
+      expect(model.errors[:query]).to contain_exactly("can't be blank")
+    end
+
+    it "is invalid with blank schema_key" do
+      model = build_subscription(schema_key: "")
+
+      model.validate
+
+      expect(model.errors[:schema_key]).to contain_exactly("can't be blank")
+    end
+
+    it "is invalid with an invalid schema_key" do
+      model = build_subscription(schema_key: "invalid")
+
+      model.validate
+
+      expect(model.errors[:query]).to contain_exactly(/Could not find a schema for schema_key 'invalid'/)
+    end
+
+    it "is invalid with an invalid query" do
+      model = build_subscription(
+        schema_key: "test",
+        query: subscription_query(query: " foo { a_field_that_does_not_exist }")
+      )
+
+      model.validate
+
+      expect(model).to_not be_valid
+      expect(model.errors[:query]).to contain_exactly(
+        "Field 'a_field_that_does_not_exist' doesn't exist on type 'testObject' (id: <new_record>)"
+      )
+    end
+
+    it "valid with a valid query" do
+      model = build_subscription(
+        schema_key: "test",
+        query: subscription_query(query: "foo { id }")
+      )
+
+      model.validate
+
+      expect(model.errors).to_not include(:query)
+    end
+  end
+
+  def build_subscription(overrides = {})
+    attrs = {
+      schema_key: "test",
+      query: subscription_query
+    }.merge(overrides)
+
+    NulogyMessageBusProducer::Subscription.new(attrs)
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb

@@ -0,0 +1,137 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscriptions::PostgresTransport do
+  context "when subscription is triggered" do
+    let(:company_uuid) { SecureRandom.uuid }
+
+    it "generates an event for a subscription" do
+      subscription = subscribe_to(
+        event_type: "testCreated",
+        topic_name: "some_topic"
+      )
+      root_object = {
+        foo: { id: "some id" },
+        company_uuid: company_uuid
+      }
+
+      trigger_event("testCreated", root_object)
+
+      event = NulogyMessageBusProducer::SubscriptionEvent.find_by!(subscription_id: subscription.id)
+      expect(event).to have_attributes(
+        partition_key: "#{subscription.subscription_group_id},#{company_uuid}",
+        topic_name: "some_topic",
+        event_json: include_json(testCreated: { foo: { id: "some id" } }),
+        company_uuid: company_uuid
+      )
+    end
+
+    it "allows configuring context" do
+      NulogyMessageBusProducer.config.context_for_subscription = lambda do |_|
+        { context_data: "some contextual information" }
+      end
+      subscription = subscribe_to(
+        event_type: "testCreated",
+        query: <<~GRAPHQL
+          foo {
+            contextData
+          }
+        GRAPHQL
+      )
+      root_object = {
+        foo: {},
+        company_uuid: company_uuid
+      }
+
+      trigger_event("testCreated", root_object)
+
+      event = NulogyMessageBusProducer::SubscriptionEvent.find_by!(subscription_id: subscription.id)
+      expect(event).to have_attributes(
+        event_json: include_json(
+          testCreated: { foo: { contextData: "some contextual information" } }
+        )
+      )
+    end
+
+    it "generates one event per subscription" do
+      subscribe_to(event_type: "testCreated")
+      subscribe_to(event_type: "testCreated")
+
+      expect do
+        root_object = {
+          foo: { id: "some id" },
+          company_uuid: company_uuid
+        }
+        trigger_event("testCreated", root_object)
+      end.to change(NulogyMessageBusProducer::SubscriptionEvent, :count).by(2)
+
+      event_data = NulogyMessageBusProducer::SubscriptionEvent.pluck(:event_json)
+      expect(event_data).to all(
+        include_json(testCreated: { foo: { id: "some id" } })
+      )
+    end
+  end
+
+  context "when the class is not registered" do
+    it "raises" do
+      expect do
+        Class.new(GraphQL::Schema) do
+          use(NulogyMessageBusProducer::Subscriptions::PostgresTransport)
+        end
+      end.to raise_error(KeyError, /The schema registry did not contain an entry/)
+    end
+  end
+
+  context "when a subscription error occurs" do
+    context "when failing with raise" do
+      before do
+        NulogyMessageBusProducer.config.producing_events_fails_with(:raise)
+      end
+
+      it "raises an exception" do
+        expect { trigger_erroneous_subscription }.to raise_error(/A subscription event could not be produced/)
+      end
+    end
+
+    context "when failing with a soft fail" do
+      before do
+        NulogyMessageBusProducer.config.producing_events_fails_with(:soft_fail)
+      end
+
+      it "does not raise an error" do
+        trigger_erroneous_subscription
+      end
+
+      it "calls the provided block" do
+        called = false
+
+        NulogyMessageBusProducer.config.producing_events_fails_with(:soft_fail) do |_|
+          called = true
+        end
+
+        trigger_erroneous_subscription
+
+        expect(called).to be(true)
+      end
+    end
+  end
+
+  def trigger_erroneous_subscription
+    event_type = "testCreated"
+
+    subscription = subscribe_to(event_type: event_type)
+    simulate_invalid_query(subscription)
+
+    trigger_event(event_type, uuid: SecureRandom.uuid, company_uuid: SecureRandom.uuid)
+  end
+
+  def simulate_invalid_query(subscription)
+    subscription.query = <<~GRAPHQL
+      query ($id: UUID!) {
+        foo (id: $id) {
+          a_field_that_does_not_exist
+        }
+      }
+    GRAPHQL
+    subscription.save(validate: false)
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb

@@ -0,0 +1,51 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscriptions::RiskySubscriptionBlocker do
+  it "blocks subscriptions with arguments" do
+    query = <<~GRAPHQL
+      foo {
+        fieldWithArguments(first: "value")
+      }
+    GRAPHQL
+
+    result = attempt_subscription(query)
+
+    expect(result).to include_json(
+      errors: [{
+        message: "Arguments may not be used:\nfieldWithArguments"
+      }]
+    )
+  end
+
+  it "blocks subscriptions which would expand lists" do
+    query = <<~GRAPHQL
+      fooList {
+        id
+      }
+    GRAPHQL
+
+    result = attempt_subscription(query)
+
+    expect(result).to include_json(
+      errors: [{
+        message: "Lists may not be queried:\nfooList"
+      }]
+    )
+  end
+
+  def attempt_subscription(query)
+    gql = <<~GRAPHQL
+      subscription {
+        testCreated (
+          subscriptionId: "#{SecureRandom.uuid}",
+          subscriptionGroupId: "#{SecureRandom.uuid}",
+          topicName: "some_topic"
+        ) {
+          #{query}
+        }
+      }
+    GRAPHQL
+
+    execute_graphql(gql, NulogyMessageBusProducer::Specs::TestSchema)
+  end
+end

data/spec/integration_spec_helper.rb

@@ -6,7 +6,9 @@ ENV["RAILS_ENV"] ||= "test"
 require File.expand_path("dummy/config/environment.rb", __dir__)
 
 require "rspec/rails"
+require "rspec/json_expectations"
 require "active_record"
+require "spec_helper"
 
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].sort.each { |f| require f }
 
@@ -32,4 +34,8 @@ RSpec.configure do |config|
   config.expect_with(:rspec) do |c|
     c.max_formatted_output_length = 1_000_000
   end
+
+  config.include(SpecUtils)
+  config.include(SqlHelpers)
+  config.include(SubscriptionHelpers)
 end