nulogy_message_bus_producer 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d983bc5429dfc09c434c773bcd93d71a6bb26fe72a14fe16c95a2d0819f1e540
+  data.tar.gz: 15b57a37e9a6975ef474826fb2156071d19f6b95c7d800e98ea39de36908ad80
+SHA512:
+  metadata.gz: 5b7971fa352fbfbbf7ec0d2f8f190eb9901a7dccdb76b121b8a20459f231e76d89b59994f29a66c90b713458b6c5e9367559a5663af78b38ef5bc09728663d68
+  data.tar.gz: 37e525876eedc66d78d3b22d5b9f47f610bfe7e08f352054cd194d50096740ab16058c1dcb4e2a260a4de2b55fe9a77bfd995162a9772c0991eb0ab4246f7151

data/README.md

@@ -0,0 +1,35 @@
+# Nulogy Message Bus Producer
+
+Nulogy's code for producing to to the Message Bus
+
+## Installation
+
+1. Add the gem to your Gemfile `gem "nulogy_message_bus_producer"`
+2. Install the migration `rake railties:install:migrations`
+
+## Usage
+
+This engine contains the classes and validations for producing messages to the Message Bus.
+
+In general, as long as a subscription inherits from `NulogyMessageBusProducer::BasePublicSubscription`, everything
+should be set up.
+
+Subscriptions will be written to `public_subscriptions` while events generated from the subscriptions will be written to
+`public_subscription_events`. Debezium then reads this table from the PG replication log and pushes them into Kafka.
+
+## Configuration
+
+| Configuration                  | Default                         | Description                                                  |
+|--------------------------------|---------------------------------|--------------------------------------------------------------|
+| subscriptions_table_name       | message_bus_subscriptions       | The name of the table for the PublicSubscription model       |
+| subscription_events_table_name | message_bus_subscription_events | The name of the table for the PublicSubscriptionEvents model |
+
+## Initializer
+
+There is an initializer (see `engine.rb`) that validates if the current `public_subscriptions.query` queries are valid
+against their appropriate GraphQL schema.
+
+## Rake Tasks
+
+`rake spec` - Run the specs
+`rake rubocop` - Run RuboCop

data/Rakefile

@@ -0,0 +1,22 @@
+#!/usr/bin/env rake
+begin
+  require "bundler/setup"
+rescue LoadError
+  puts "You must `gem install bundler` and `bundle install` to run rake tasks"
+end
+
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+Bundler::GemHelper.install_tasks
+
+require "rspec/core"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: [:spec, :rubocop]

data/db/migrate/20200611150212_create_public_subscriptions_and_events_tables.rb

@@ -0,0 +1,25 @@
+class CreatePublicSubscriptionsAndEventsTables < ActiveRecord::Migration[5.2]
+  def change
+    # TODO: You may need to tweak this setup to suit your app's database
+    enable_extension "uuid-ossp"
+
+    create_table NulogyMessageBusProducer.subscriptions_table_name, id: :uuid, default: nil do |t|
+      t.uuid :subscription_group_id, null: false
+      t.string :event_type, null: false, index: { name: "index_nulogy_mb_producer_subscriptions_on_event_type" }
+      t.string :topic_name, null: false
+      t.string :query, null: false
+      t.text :schema_key, null: false
+
+      t.timestamps
+    end
+
+    create_table NulogyMessageBusProducer.subscription_events_table_name, id: :uuid, default: nil do |t|
+      t.uuid :public_subscription_id, null: false
+      t.string :partition_key, null: false
+      t.string :topic_name, null: false
+      t.uuid :tenant_id, null: false
+      t.json :event_json, null: false
+      t.column :created_at, :datetime, index: { name: "index_nulogy_mb_producer_subscription_events_on_created_at" }
+    end
+  end
+end

data/lib/nulogy_message_bus_producer.rb

@@ -0,0 +1,112 @@
+require "graphql"
+require "strong_migrations"
+
+require "nulogy_message_bus_producer/version"
+require "nulogy_message_bus_producer/engine"
+
+# The gem and its configuration
+module NulogyMessageBusProducer
+  mattr_reader :registered_schemas, default: {}
+
+  def self.subscriptions_table_name=(table_name)
+    NulogyMessageBusProducer::PublicSubscription.table_name = table_name
+  end
+
+  def self.subscriptions_table_name
+    NulogyMessageBusProducer::PublicSubscription.table_name
+  end
+
+  def self.subscription_events_table_name=(table_name)
+    NulogyMessageBusProducer::PublicSubscriptionEvent.table_name = table_name
+  end
+
+  def self.subscription_events_table_name
+    NulogyMessageBusProducer::PublicSubscriptionEvent.table_name
+  end
+
+  def self.register_schema(schema_key, schema_name)
+    registered_schemas[schema_key] = schema_name
+  end
+
+  def self.resolve_schema(schema_key)
+    registered_schemas.fetch(schema_key) do
+      raise KeyError, <<~MESSAGE unless block_given?
+        The schema registry did not contain an entry for the schema key '#{schema_key}'.
+
+        Please register the schema first with `NulogyMessageBusProducer.schemas.register(schema_key, schema)`
+      MESSAGE
+
+      yield
+    end.constantize
+  end
+
+  def self.resolve_schema_key(schema)
+    registered_schemas.invert.fetch(schema.class.name) do
+      raise KeyError, <<~MESSAGE
+        The schema registry did not contain an entry for the schema '#{schema.class.name}'.
+
+        Please register the schema first with `NulogyMessageBusProducer.schemas.register(schema_key, schema)`
+      MESSAGE
+    end
+  end
+
+  def self.validate_existing!
+    return unless Db.exists? && Db.subscriptions_exist?
+
+    validator = NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator.new
+
+    return if validator.validate
+
+    raise <<~MESSAGE
+      #{'#' * 80}
+
+      The GraphQL queries stored in public_subscriptions.query must always remain valid against the current schema.
+      If one is not valid, a domain event firing could fail and users would not be able to perform critical duties
+      (e.g. adding production).
+
+      Note that you may experience errors in different environments, e.g. NA fails but EU passes.
+      If you can quickly fix the data, it is recommended you fix the data and roll forward for the failing
+      environment.
+
+      If you need more time to plan or execute an action, it is advised that you rollback the deployment
+      for all environments so that they are consistent.
+
+      The following errors were found while validating those queries against its respective schema:
+
+      #{validator.errors.join("\n")}
+
+      #{'#' * 80}
+    MESSAGE
+  end
+
+  # A private helper module to check database connectivity and setup for the above initializer
+  module Db
+    module_function
+
+    def exists?
+      ActiveRecord::Base.connection
+    rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad
+      false
+    else
+      true
+    end
+
+    def subscriptions_exist?
+      ActiveRecord::Base
+        .connection
+        .exec_query(
+          "SELECT tablename FROM pg_tables WHERE tablename = '#{NulogyMessageBusProducer.subscriptions_table_name}' AND schemaname = 'public';"
+        ).present?
+    rescue # rubocop:disable Style/RescueStandardError
+      false
+    end
+  end
+  private_constant :Db
+end
+
+require "nulogy_message_bus_producer/application_record"
+require "nulogy_message_bus_producer/base_public_subscription"
+require "nulogy_message_bus_producer/postgres_public_subscriptions"
+require "nulogy_message_bus_producer/public_subscription"
+require "nulogy_message_bus_producer/public_subscription_event"
+require "nulogy_message_bus_producer/subscriber_graphql_schema_validator"

data/lib/nulogy_message_bus_producer/application_record.rb

@@ -0,0 +1,6 @@
+module NulogyMessageBusProducer
+  # ApplicationRecord is a super-class of all your models to add shared behaviour
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/lib/nulogy_message_bus_producer/base_public_subscription.rb

@@ -0,0 +1,13 @@
+module NulogyMessageBusProducer
+  # This base class contains the fields required to create a subscription.
+  # For example, for a subscription to a model called AggregateRoot:
+  #
+  #  class AggregateRootCreated < NulogyMessageBusProducer::BasePublicSubscription
+  #    field :aggregate_root, Domain::Public::Types::AggregateRootType, null: true
+  #  end
+  class BasePublicSubscription < GraphQL::Schema::Subscription
+    argument :subscription_id, ID, required: true
+    argument :subscription_group_id, ID, required: true
+    argument :topic_name, String, required: true
+  end
+end

data/lib/nulogy_message_bus_producer/engine.rb

@@ -0,0 +1,6 @@
+module NulogyMessageBusProducer
+  # The Rails engine
+  class Engine < Rails::Engine
+    isolate_namespace NulogyMessageBusProducer
+  end
+end

data/lib/nulogy_message_bus_producer/postgres_public_subscriptions.rb

@@ -0,0 +1,102 @@
+module NulogyMessageBusProducer
+  # Subscription class to `use` when developing Message Bus-backed subscriptions
+  # For example,
+  #
+  #  class SomeSchema < GraphQL::Schema
+  #    ...
+  #    use NulogyMessageBusProducer::PostgresPublicSubscriptions
+  #  end
+  #
+  # It expects that schema to already be registered, or will raise an error.
+  #
+  # NulogyMessageBusProducer.register_schema("some_schema", "SomeSchema")
+  class PostgresPublicSubscriptions < GraphQL::Subscriptions
+    def initialize(options = {})
+      super
+
+      @schema_key = NulogyMessageBusProducer.resolve_schema_key(options.fetch(:schema))
+    end
+
+    # This method is copied from GraphQL::Subscriptions and customized.
+    # Check for changes in the superclass when upgrading the graphql gem.
+    def execute(subscription_id, event, object) # rubocop:disable Metrics/MethodLength
+      query_data = read_subscription(subscription_id)
+      query_string = query_data.fetch(:query_string)
+      variables = { id: object[:uuid] }
+      context = object[:context]
+      operation_name = query_data.fetch(:operation_name)
+      result = @schema.execute(
+        query: query_string,
+        context: context,
+        subscription_topic: event.topic,
+        operation_name: operation_name,
+        variables: variables,
+        root_value: object
+      )
+      deliver(subscription_id, result)
+    rescue GraphQL::Schema::Subscription::NoUpdateError
+      # This update was skipped in user code; do nothing.
+    rescue GraphQL::Schema::Subscription::UnsubscribedError
+      delete_subscription(subscription_id)
+    end
+
+    def each_subscription_id(event)
+      Models::PublicSubscription.where(event_type: event.name, schema_key: @schema_key).each do |subscription|
+        yield subscription.id
+      end
+    end
+
+    def read_subscription(subscription_id)
+      query_string = Models::PublicSubscription.find_by(id: subscription_id).query
+
+      {
+        query_string: query_string,
+        operation_name: nil
+      }
+    end
+
+    def deliver(subscription_id, result)
+      tenant_id = result.query.context.object[:tenant_id]
+      subscription = Models::PublicSubscription.find_by(id: subscription_id)
+
+      Models::PublicSubscriptionEvent.create_or_update(
+        id: SecureRandom.uuid,
+        public_subscription_id: subscription_id,
+        partition_key: "#{subscription.subscription_group_id},#{tenant_id}",
+        tenant_id: tenant_id,
+        event_json: result.to_h["data"],
+        topic_name: subscription.topic_name
+      )
+    end
+
+    def write_subscription(query, events)
+      events.each do |event|
+        Models::PublicSubscription.create_or_update(
+          id: event.arguments[:subscription_id],
+          subscription_group_id: event.arguments[:subscription_group_id],
+          event_type: event.name,
+          schema_key: @schema_key,
+          query: convert_subscription_result_query_to_regular_query(query, event),
+          topic_name: event.arguments[:topic_name]
+        )
+      end
+    end
+
+    def delete_subscription(subscription_id)
+      Models::PublicSubscription.find_by(id: subscription_id).destroy
+    end
+
+    private
+
+    def convert_subscription_result_query_to_regular_query(query, event)
+      selections = query.document.definitions.first.selections
+
+      event_selection = selections.find { |e| e.name == event.name }
+
+      # TODO: This assumes the query_type.rb takes an argument called 'id'.  This should check the query to lookup the argument name
+      # TODO: Can we do this by manipulating GQL objects?
+      inner_query = event_selection.selections.first.to_query_string.sub("{", "(id: $id) {")
+      "query ($id: UUID!) { #{inner_query} }"
+    end
+  end
+end

data/lib/nulogy_message_bus_producer/public_subscription.rb

@@ -0,0 +1,27 @@
+module NulogyMessageBusProducer
+  # This model saves all subscriptions to external systems.
+  # An external system can subscribe to events and specify the shape of data it would like to receive for the event.
+  class PublicSubscription < ApplicationRecord
+    self.table_name = :message_bus_subscriptions
+
+    # Run our validator with familar syntax in this model
+    class ValidForSchemaValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, _value)
+        validator = NulogyMessageBusProducer::SubscriberGraphqlSchemaValidator.new
+
+        validator.validate(record)
+
+        validator.errors.each { |e| record.errors.add(attribute, e) }
+      end
+    end
+
+    validates :query, presence: true, valid_for_schema: true
+    validates :schema_key, presence: true
+
+    def self.create_or_update(attrs)
+      find_or_initialize_by(id: attrs[:id]).tap do |model|
+        model.update!(attrs)
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_producer/public_subscription_event.rb

@@ -0,0 +1,13 @@
+module NulogyMessageBusProducer
+  # A model that contains the event data for a particular subscription
+  # It is simply saved in the database and shipped to Kafka by Debezium
+  class PublicSubscriptionEvent < ApplicationRecord
+    self.table_name = :message_bus_subscription_events
+
+    def self.create_or_update(attrs)
+      find_or_initialize_by(id: attrs[:id]).tap do |model|
+        model.update!(attrs)
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator.rb

@@ -0,0 +1,45 @@
+module NulogyMessageBusProducer
+  # A custom validator that checks that the provided (or all) subscriptions have a query that is valid for its
+  # configured schema. Schemas must be registered with a `schema_key`, that is persisted in the database.
+  # It ties the subscription to a particular schema.
+  #
+  # This validator is run as part of an initializer as a last ditch effort to verify that the stored queries in the
+  # database are valid against the deployed schema, so that when events are generated in the system, they are always
+  # sucessfully created.
+  class SubscriberGraphqlSchemaValidator
+    attr_reader :errors
+
+    def initialize
+      @errors = []
+    end
+
+    def validate(subscription_or_subscriptions = NulogyMessageBusProducer::PublicSubscription.all)
+      Array(subscription_or_subscriptions).each do |subscription|
+        schema = find_schema(subscription)
+        next unless schema
+
+        gql_errors = schema.validate(subscription.query)
+        errors = gql_errors.map { |e| "#{e.message} #{display_id(subscription.id)}" }
+
+        @errors.concat(errors)
+      end
+
+      @errors.empty?
+    end
+
+    private
+
+    def find_schema(subscription)
+      NulogyMessageBusProducer.resolve_schema(subscription.schema_key) do
+        @errors << "Could not find a schema for schema_key '#{subscription.schema_key}' #{display_id(subscription.id)}"
+        nil
+      end
+    end
+
+    def display_id(id)
+      normalized = id.presence || "<new_record>"
+
+      "(id: #{normalized})"
+    end
+  end
+end

data/lib/nulogy_message_bus_producer/version.rb

@@ -0,0 +1,3 @@
+module NulogyMessageBusProducer
+  VERSION = "1.0.2".freeze
+end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative 'config/application'
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,4 @@
+
+//= link_tree ../images
+//= link_directory ../javascripts .js
+//= link_directory ../stylesheets .css