nulogy_message_bus_producer 2.0.0 → 3.2.1

data/spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb

@@ -0,0 +1,141 @@
+require "integration_spec_helper"
+require "net/http"
+
+RSpec.describe NulogyMessageBusProducer::RepopulateReplicationSlots do
+  let(:company_uuid) { SecureRandom.uuid }
+  let(:number_of_messages) { 100 }
+
+  let(:kafka_bootstrap_servers) { "host.docker.internal:39092" }
+  let(:kafka_connect) { KafkaConnect.new("http://localhost:8083", "ruby_specs") }
+  let(:replication_slot_name) { "rspec_replication_slot" }
+  let(:topic_name) { "repopulate_replication_slot_tests" }
+
+  before do
+    cleanup_everything
+  end
+
+  it "generates events" do
+    Kafka.create_topic(topic_name)
+    consumer = Kafka.setup_kafka_consumer(topic_name)
+
+    without_transaction do
+      subscribe_to(event_type: "testCreated", topic_name: topic_name)
+
+      number_of_messages.times { |n| create_event(uuid(n)) }
+
+      start_debezium
+
+      described_class.repopulate
+    end
+
+    message_payloads = Kafka.wait_for_messages(consumer).map(&:payload)
+    expect(message_payloads.count).to eq(number_of_messages)
+    matcher = Array.new(number_of_messages) { |n| include(uuid(n)) }
+    expect(message_payloads).to match(matcher)
+  end
+
+  def cleanup_everything
+    truncate_db
+    begin
+      Kafka.delete_topic(topic_name)
+    rescue StandardError
+      nil
+    end
+    kafka_connect.delete
+    wait_for_replication_slot_cleanup(replication_slot_name)
+  end
+
+  def create_event(entity_uuid)
+    root_object = {
+      company_uuid: company_uuid,
+      foo: {
+        id: entity_uuid
+      }
+    }
+    trigger_event("testCreated", root_object)
+  end
+
+  def start_debezium # rubocop:disable Metrics/AbcSize
+    config = build_debezium_config
+
+    response = kafka_connect.configure(config)
+
+    expect(response.code).to eq("201"), <<~MESSAGE
+      Creating a Debezium config in Kafka Connect has failed. HTTP Request returned:
+      Code: #{response.code}
+      #{JSON.parse(response.body).pretty_inspect}
+      #{config.pretty_inspect}
+    MESSAGE
+    wait_for_tasks_to_start(kafka_connect)
+    wait_for_replication_slot(replication_slot_name)
+  end
+
+  def wait_for_tasks_to_start(kafka_connect)
+    SpecUtils.wait_for(attempts: 10, interval: 0.5) do
+      tasks = kafka_connect.status[:tasks]
+      next false if tasks.blank?
+
+      expect(tasks.all? { |task| task[:state] == "RUNNING" }).to eq(true), <<~MESSAGE
+        Expected the Kafka Connect tasks to be running. Instead found:
+        #{tasks.pretty_inspect}
+      MESSAGE
+    end
+  end
+
+  def build_debezium_config # rubocop:disable Metrics/MethodLength
+    db_config = Rails.configuration.database_configuration[Rails.env]
+    events_table = NulogyMessageBusProducer::SubscriptionEvent.table_name
+
+    {
+      "bootstrap.servers": kafka_bootstrap_servers,
+
+      "database.dbname": db_config["database"],
+      "database.hostname": db_config["host"] == "localhost" ? "host.docker.internal" : db_config["host"],
+      "database.password": db_config["password"],
+      "database.port": db_config["port"] || 5432,
+      "database.server.name": "test-environment",
+      "database.user": db_config["username"],
+      "slot.name": replication_slot_name,
+
+      "behavior.on.null.values": "delete",
+      "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+      "database.initial.statements": <<~SQL,
+        DO $$BEGIN
+          IF NOT EXISTS(SELECT FROM pg_publication WHERE pubname = 'debezium_public_events')
+          THEN CREATE PUBLICATION debezium_public_events FOR TABLE #{events_table} WITH (publish = 'insert');; END IF;; END$$;
+      SQL
+      "errors.log.enable": "true",
+      "errors.log.include.messages": "true",
+      "heartbeat.interval.ms": "30000",
+      "plugin.name": "pgoutput",
+      "publication.name": "debezium_public_events",
+      "slot.drop.on.stop": "true",
+      "snapshot.mode": "never",
+      "table.whitelist": "public.#{events_table}",
+
+      "transforms": "requireTopicName,unwrap,extractTopicName,extractPartitionKey,removeFields",
+
+      "transforms.requireTopicName.type": "io.confluent.connect.transforms.Filter$Value",
+      "transforms.requireTopicName.filter.condition": "$.after.topic_name",
+      "transforms.requireTopicName.filter.type": "include",
+      "transforms.requireTopicName.missing.or.null.behavior": "exclude",
+
+      "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
+      "transforms.unwrap.drop.tombstones": "true",
+
+      "transforms.extractTopicName.type": "io.confluent.connect.transforms.ExtractTopic$Value",
+      "transforms.extractTopicName.field": "topic_name",
+
+      "transforms.extractPartitionKey.type": "org.apache.kafka.connect.transforms.ValueToKey",
+      "transforms.extractPartitionKey.fields": "partition_key",
+
+      "transforms.removeFields.type": "org.apache.kafka.connect.transforms.ReplaceField$Value",
+      "transforms.removeFields.blacklist": "topic_name,partition_key",
+
+      "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+      "key.converter.schemas.enable": "false",
+      "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+      "value.converter.schemas.enable": "false"
+    }
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator_spec.rb

@@ -0,0 +1,49 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator do
+  subject(:validator) { described_class.new }
+
+  describe "#validate" do
+    context "when a valid query is present" do
+      it "return true" do
+        subscribe_to(query: <<~GRAPHQL)
+          foo {
+            id
+          }
+        GRAPHQL
+
+        expect(validator.validate).to be(true)
+      end
+    end
+
+    context "when an invalid query is present" do
+      let(:subscription_with_error) do
+        subscription = subscribe_to(query: <<~GRAPHQL)
+          foo {
+            id
+          }
+        GRAPHQL
+
+        subscription.query.gsub!(/\bid\b/, "a_field_that_does_not_exist")
+        subscription.save(validate: false)
+        subscription
+      end
+
+      it "returns false" do
+        subscription_with_error
+
+        expect(validator.validate).to be(false)
+      end
+
+      it "has errors" do
+        subscription_with_error
+
+        validator.validate
+
+        expect(validator.errors).to contain_exactly(
+          "Field 'a_field_that_does_not_exist' doesn't exist on type 'testObject' (id: #{subscription_with_error.id})"
+        )
+      end
+    end
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb

@@ -0,0 +1,61 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscription do
+  context "when validating" do
+    it "is invalid with a blank query" do
+      model = build_subscription(query: "")
+
+      model.validate
+
+      expect(model.errors[:query]).to contain_exactly("can't be blank")
+    end
+
+    it "is invalid with blank schema_key" do
+      model = build_subscription(schema_key: "")
+
+      model.validate
+
+      expect(model.errors[:schema_key]).to contain_exactly("can't be blank")
+    end
+
+    it "is invalid with an invalid schema_key" do
+      model = build_subscription(schema_key: "invalid")
+
+      model.validate
+
+      expect(model.errors[:query]).to contain_exactly(/Could not find a schema for schema_key 'invalid'/)
+    end
+
+    it "is invalid with an invalid query" do
+      model = build_subscription(
+        query: subscription_query(query: "foo { a_field_that_does_not_exist }")
+      )
+
+      model.validate
+
+      expect(model).not_to be_valid
+      expect(model.errors[:query]).to contain_exactly(
+        "Field 'a_field_that_does_not_exist' doesn't exist on type 'testObject' (id: <new_record>)"
+      )
+    end
+
+    it "valid with a valid query" do
+      model = build_subscription(
+        query: subscription_query(query: "foo { id }")
+      )
+
+      model.validate
+
+      expect(model.errors).not_to include(:query)
+    end
+  end
+
+  def build_subscription(overrides = {})
+    attrs = {
+      schema_key: "test",
+      query: subscription_query
+    }.merge(overrides)
+
+    NulogyMessageBusProducer::Subscription.new(attrs)
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/no_variables_spec.rb

@@ -0,0 +1,46 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscriptions::NoVariables do
+  it "blocks subscriptions with variables" do
+    query = <<~SUBSCRIPTION
+      subscription($sub_id: ID!, $sg_id: ID!, $topic: String!)  {
+        testCreated(subscriptionId: $sub_id, subscriptionGroupId: $sg_id, topicName: $topic) {
+          foo { id }
+        }
+      }
+    SUBSCRIPTION
+
+    result = execute_graphql(
+      query,
+      NulogyMessageBusProducer::Specs::TestSchema,
+      variables: {
+        sub_id: SecureRandom.uuid,
+        sg_id: SecureRandom.uuid,
+        topic: "test_topic"
+      }
+    )
+
+    expect(result).to include_json(
+      errors: [{ message: include("Subscriptions should not be created with arguments") }]
+    )
+  end
+
+  it "does not block subscriptions without variables" do
+    query = <<~SUBSCRIPTION
+      subscription  {
+        testCreated(
+          subscriptionId: "#{SecureRandom.uuid}",
+          subscriptionGroupId: "#{SecureRandom.uuid}",
+          topicName: "test_topic"
+        ) { foo { id } }
+      }
+    SUBSCRIPTION
+
+    result = execute_graphql(
+      query,
+      NulogyMessageBusProducer::Specs::TestSchema
+    )
+
+    expect(result).not_to include(:errors)
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb

@@ -0,0 +1,135 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscriptions::PostgresTransport do
+  context "when subscription is triggered" do
+    let(:company_uuid) { SecureRandom.uuid }
+
+    it "generates an event for a subscription" do
+      subscription = subscribe_to(
+        event_type: "testCreated",
+        topic_name: "some_topic"
+      )
+      root_object = {
+        foo: { id: "some id" },
+        company_uuid: company_uuid
+      }
+
+      trigger_event("testCreated", root_object)
+
+      event = NulogyMessageBusProducer::SubscriptionEvent.find_by!(subscription_id: subscription.id)
+      expect(event).to have_attributes(
+        partition_key: "#{subscription.subscription_group_id},#{company_uuid}",
+        topic_name: "some_topic",
+        event_json: include_json(testCreated: { foo: { id: "some id" } }),
+        company_uuid: company_uuid
+      )
+    end
+
+    it "allows configuring context" do
+      NulogyMessageBusProducer.config.context_for_subscription = lambda do |_|
+        { context_data: "some contextual information" }
+      end
+      subscription = subscribe_to(
+        event_type: "testCreated",
+        query: <<~GRAPHQL
+          foo {
+            contextData
+          }
+        GRAPHQL
+      )
+      root_object = {
+        foo: {},
+        company_uuid: company_uuid
+      }
+
+      trigger_event("testCreated", root_object)
+
+      event = NulogyMessageBusProducer::SubscriptionEvent.find_by!(subscription_id: subscription.id)
+      expect(event).to have_attributes(
+        event_json: include_json(testCreated: { foo: { contextData: "some contextual information" } })
+      )
+    end
+
+    it "generates one event per subscription" do
+      subscribe_to(event_type: "testCreated")
+      subscribe_to(event_type: "testCreated")
+
+      expect do
+        root_object = {
+          foo: { id: "some id" },
+          company_uuid: company_uuid
+        }
+        trigger_event("testCreated", root_object)
+      end.to change(NulogyMessageBusProducer::SubscriptionEvent, :count).by(2)
+
+      event_data = NulogyMessageBusProducer::SubscriptionEvent.pluck(:event_json)
+      expect(event_data).to all(
+        include_json(testCreated: { foo: { id: "some id" } })
+      )
+    end
+  end
+
+  context "when the class is not registered" do
+    it "raises" do
+      expect do
+        Class.new(GraphQL::Schema) do
+          use(NulogyMessageBusProducer::Subscriptions::PostgresTransport)
+        end
+      end.to raise_error(KeyError, /The schema registry did not contain an entry/)
+    end
+  end
+
+  context "when a subscription error occurs" do
+    context "when failing with raise" do
+      before do
+        NulogyMessageBusProducer.config.producing_events_fails_with(:raise)
+      end
+
+      it "raises an exception" do
+        expect { trigger_erroneous_subscription }.to raise_error(/A subscription event could not be produced/)
+      end
+    end
+
+    context "when failing with a soft fail" do
+      before do
+        NulogyMessageBusProducer.config.producing_events_fails_with(:soft_fail)
+      end
+
+      it "does not raise an error" do
+        trigger_erroneous_subscription
+      end
+
+      it "calls the provided block" do
+        called = false
+
+        NulogyMessageBusProducer.config.producing_events_fails_with(:soft_fail) do |_|
+          called = true
+        end
+
+        trigger_erroneous_subscription
+
+        expect(called).to be(true)
+      end
+    end
+  end
+
+  def trigger_erroneous_subscription
+    event_type = "testCreated"
+
+    subscription = subscribe_to(event_type: event_type)
+    simulate_invalid_query(subscription)
+
+    trigger_event(event_type, uuid: SecureRandom.uuid, company_uuid: SecureRandom.uuid)
+  end
+
+  def simulate_invalid_query(subscription)
+    subscription.query = <<~GRAPHQL
+      query ($id: UUID!) {
+        foo (id: $id) {
+          a_field_that_does_not_exist
+        }
+      }
+    GRAPHQL
+    subscription.save(validate: false)
+  end
+end

data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb

@@ -0,0 +1,49 @@
+require "integration_spec_helper"
+
+RSpec.describe NulogyMessageBusProducer::Subscriptions::RiskySubscriptionBlocker do
+  it "blocks subscriptions with arguments" do
+    query = <<~GRAPHQL
+      foo {
+        fieldWithArguments(first: "value")
+      }
+    GRAPHQL
+
+    result = attempt_subscription(query)
+
+    expect(result).to include_json(
+      errors: [{
+        message: "Arguments may not be used:\nfieldWithArguments"
+      }]
+    )
+  end
+
+  it "blocks subscriptions which would expand lists" do
+    query = <<~GRAPHQL
+      fooList {
+        id
+      }
+    GRAPHQL
+
+    result = attempt_subscription(query)
+
+    expect(result).to include_json(
+      errors: [{
+        message: "Lists may not be queried:\nfooList"
+      }]
+    )
+  end
+
+  def attempt_subscription(query)
+    execute_graphql(<<~GRAPHQL, NulogyMessageBusProducer::Specs::TestSchema)
+      subscription {
+        testCreated (
+          subscriptionId: "#{SecureRandom.uuid}",
+          subscriptionGroupId: "#{SecureRandom.uuid}",
+          topicName: "some_topic"
+        ) {
+          #{query}
+        }
+      }
+    GRAPHQL
+  end
+end