nulogy_message_bus_producer 3.2.1 → 3.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +82 -22
- data/Rakefile +2 -4
- data/db/migrate/20200611150212_create_public_subscriptions_and_events_tables.rb +2 -2
- data/lib/nulogy_message_bus_producer.rb +15 -8
- data/lib/nulogy_message_bus_producer/base_subscription.rb +1 -1
- data/lib/nulogy_message_bus_producer/config.rb +25 -1
- data/lib/nulogy_message_bus_producer/configuration/query_parser.rb +71 -0
- data/lib/nulogy_message_bus_producer/self_serve_subscription.rb +18 -0
- data/lib/nulogy_message_bus_producer/subscriptions/configured_subscription.rb +14 -0
- data/lib/nulogy_message_bus_producer/subscriptions/finder.rb +40 -0
- data/lib/nulogy_message_bus_producer/subscriptions/postgres_transport.rb +13 -6
- data/lib/nulogy_message_bus_producer/subscriptions/query_validator.rb +47 -0
- data/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker.rb +0 -5
- data/lib/nulogy_message_bus_producer/subscriptions/valid_for_schema_validator.rb +14 -0
- data/lib/nulogy_message_bus_producer/version.rb +1 -1
- data/spec/dummy/Rakefile +1 -1
- data/spec/dummy/app/mailers/application_mailer.rb +2 -2
- data/spec/dummy/bin/bundle +2 -2
- data/spec/dummy/bin/rails +3 -3
- data/spec/dummy/bin/rake +2 -2
- data/spec/dummy/bin/setup +14 -16
- data/spec/dummy/bin/update +10 -10
- data/spec/dummy/bin/yarn +13 -7
- data/spec/dummy/config.ru +2 -1
- data/spec/dummy/config/application.rb +11 -6
- data/spec/dummy/config/boot.rb +2 -4
- data/spec/dummy/config/cable.yml +2 -2
- data/spec/dummy/config/database.yml +2 -2
- data/spec/dummy/config/environment.rb +1 -1
- data/spec/dummy/config/environments/development.rb +29 -7
- data/spec/dummy/config/environments/production.rb +50 -20
- data/spec/dummy/config/environments/test.rb +25 -8
- data/spec/dummy/config/initializers/assets.rb +2 -2
- data/spec/dummy/config/initializers/backtrace_silencers.rb +4 -3
- data/spec/dummy/config/initializers/content_security_policy.rb +30 -0
- data/spec/dummy/config/initializers/filter_parameter_logging.rb +3 -1
- data/spec/dummy/config/initializers/new_framework_defaults_6_1.rb +67 -0
- data/spec/dummy/config/initializers/permissions_policy.rb +11 -0
- data/spec/dummy/config/locales/en.yml +1 -1
- data/spec/dummy/config/puma.rb +3 -3
- data/spec/dummy/config/spring.rb +2 -2
- data/spec/dummy/config/storage.yml +34 -0
- data/spec/dummy/db/schema.rb +0 -2
- data/spec/dummy/log/development.log +317 -0
- data/spec/dummy/log/test.log +13164 -0
- data/spec/integration/lib/nulogy_message_bus_producer/config_spec.rb +37 -0
- data/spec/integration/lib/nulogy_message_bus_producer/repopulate_replication_slots_spec.rb +6 -7
- data/spec/integration/lib/nulogy_message_bus_producer/subscription_spec.rb +3 -57
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/finder_spec.rb +54 -0
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/no_variables_spec.rb +1 -1
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/postgres_transport_spec.rb +100 -54
- data/spec/integration/lib/nulogy_message_bus_producer/{subscriber_graphql_schema_validator_spec.rb → subscriptions/query_validator_spec.rb} +3 -3
- data/spec/integration/lib/nulogy_message_bus_producer/subscriptions/risky_subscription_blocker_spec.rb +0 -16
- data/spec/integration_spec_helper.rb +0 -6
- data/spec/nulogy_message_bus_producer/configuration/query_parser_spec.rb +58 -0
- data/spec/nulogy_message_bus_producer/subscriptions/subscription_spec.rb +9 -0
- data/spec/spec_helper.rb +26 -1
- data/spec/support/kafka.rb +9 -10
- data/spec/support/kafka_connect.rb +1 -1
- data/spec/support/shared_examples/subscription_validations.rb +77 -0
- data/spec/support/skip.rb +9 -0
- data/spec/support/sql_helpers.rb +1 -1
- data/spec/support/subscription_helpers.rb +22 -4
- data/spec/support/test_graphql_schema.rb +6 -2
- metadata +131 -79
- data/lib/nulogy_message_bus_producer/subscriber_graphql_schema_validator.rb +0 -45
- data/lib/nulogy_message_bus_producer/subscription.rb +0 -28
@@ -12,9 +12,9 @@ module NulogyMessageBusProducer
|
|
12
12
|
#
|
13
13
|
# NulogyMessageBusProducer.register_schema("some_schema", "SomeSchema")
|
14
14
|
class PostgresTransport < GraphQL::Subscriptions
|
15
|
-
def initialize(
|
15
|
+
def initialize(schema:, **_rest)
|
16
16
|
super
|
17
|
-
@schema_key = NulogyMessageBusProducer.resolve_schema_key(
|
17
|
+
@schema_key = NulogyMessageBusProducer.resolve_schema_key(schema)
|
18
18
|
end
|
19
19
|
|
20
20
|
def each_subscription_id(event)
|
@@ -24,7 +24,7 @@ module NulogyMessageBusProducer
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def read_subscription(subscription_id)
|
27
|
-
subscription =
|
27
|
+
subscription = find_subscription(subscription_id)
|
28
28
|
context = NulogyMessageBusProducer.context_for_subscription(subscription)
|
29
29
|
|
30
30
|
{
|
@@ -50,7 +50,7 @@ module NulogyMessageBusProducer
|
|
50
50
|
|
51
51
|
def write_subscription(query, events)
|
52
52
|
events.each do |event|
|
53
|
-
|
53
|
+
SelfServeSubscription.create_or_update(
|
54
54
|
id: event.arguments.fetch(:subscription_id),
|
55
55
|
subscription_group_id: event.arguments.fetch(:subscription_group_id),
|
56
56
|
event_type: event.name,
|
@@ -61,15 +61,16 @@ module NulogyMessageBusProducer
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
+
# TODO: how is this invoked?
|
64
65
|
def delete_subscription(subscription_id)
|
65
|
-
|
66
|
+
SelfServeSubscription.find_by(id: subscription_id).destroy
|
66
67
|
end
|
67
68
|
|
68
69
|
private
|
69
70
|
|
70
71
|
def create_event(subscription_id, result)
|
71
72
|
company_uuid = result.query.context.object[:company_uuid]
|
72
|
-
subscription =
|
73
|
+
subscription = find_subscription(subscription_id)
|
73
74
|
|
74
75
|
SubscriptionEvent.create_or_update(
|
75
76
|
id: SecureRandom.uuid,
|
@@ -80,6 +81,12 @@ module NulogyMessageBusProducer
|
|
80
81
|
topic_name: subscription.topic_name
|
81
82
|
)
|
82
83
|
end
|
84
|
+
|
85
|
+
def find_subscription(id)
|
86
|
+
Subscriptions::Finder
|
87
|
+
.new(NulogyMessageBusProducer.config)
|
88
|
+
.find(id)
|
89
|
+
end
|
83
90
|
end
|
84
91
|
end
|
85
92
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module NulogyMessageBusProducer
|
2
|
+
module Subscriptions
|
3
|
+
# A custom validator that checks that the provided (or all) subscriptions have a query that is valid for its
|
4
|
+
# configured schema. Schemas must be registered with a `schema_key`, that is persisted in the database.
|
5
|
+
# It ties the subscription to a particular schema.
|
6
|
+
#
|
7
|
+
# This validator is run as part of an initializer as a last ditch effort to verify that the stored queries in the
|
8
|
+
# database are valid against the deployed schema, so that when events are generated in the system, they are always
|
9
|
+
# sucessfully created.
|
10
|
+
class QueryValidator
|
11
|
+
attr_reader :errors
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@errors = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def validate(subscription_or_subscriptions = NulogyMessageBusProducer::SelfServeSubscription.all)
|
18
|
+
Array(subscription_or_subscriptions).each do |subscription|
|
19
|
+
schema = find_schema(subscription)
|
20
|
+
next unless schema
|
21
|
+
|
22
|
+
gql_errors = schema.validate(subscription.query)
|
23
|
+
errors = gql_errors.map { |e| "#{e.message} #{display_id(subscription.id)}" }
|
24
|
+
|
25
|
+
@errors.concat(errors)
|
26
|
+
end
|
27
|
+
|
28
|
+
@errors.empty?
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def find_schema(subscription)
|
34
|
+
NulogyMessageBusProducer.resolve_schema(subscription.schema_key) do
|
35
|
+
@errors << "Could not find a schema for schema_key '#{subscription.schema_key}' #{display_id(subscription.id)}"
|
36
|
+
nil
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def display_id(id)
|
41
|
+
normalized = id.presence || "<new_record>"
|
42
|
+
|
43
|
+
"(id: #{normalized})"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -43,11 +43,6 @@ module NulogyMessageBusProducer
|
|
43
43
|
error_messages << "Arguments may not be used:\n#{node_names}"
|
44
44
|
end
|
45
45
|
|
46
|
-
if @nodes_are_lists.any?
|
47
|
-
node_names = @nodes_are_lists.map(&:name).join("\n")
|
48
|
-
error_messages << "Lists may not be queried:\n#{node_names}"
|
49
|
-
end
|
50
|
-
|
51
46
|
GraphQL::AnalysisError.new(error_messages.join("\n\n")) if error_messages.any?
|
52
47
|
end
|
53
48
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module NulogyMessageBusProducer
|
2
|
+
module Subscriptions
|
3
|
+
class ValidForSchemaValidator < ActiveModel::EachValidator
|
4
|
+
def validate_each(record, attribute, _value)
|
5
|
+
return if record.schema_key.blank? || record.query.blank?
|
6
|
+
|
7
|
+
validator = NulogyMessageBusProducer::Subscriptions::QueryValidator.new
|
8
|
+
|
9
|
+
validator.validate(record)
|
10
|
+
validator.errors.each { |e| record.errors.add(attribute, e) }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/spec/dummy/Rakefile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Add your own tasks in files placed in lib/tasks ending in .rake,
|
2
2
|
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
|
3
3
|
|
4
|
-
require_relative
|
4
|
+
require_relative "config/application"
|
5
5
|
|
6
6
|
Rails.application.load_tasks
|
data/spec/dummy/bin/bundle
CHANGED
@@ -1,3 +1,3 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
ENV[
|
3
|
-
load Gem.bin_path(
|
2
|
+
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
|
3
|
+
load Gem.bin_path("bundler", "bundle")
|
data/spec/dummy/bin/rails
CHANGED
@@ -1,4 +1,4 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
APP_PATH = File.expand_path(
|
3
|
-
require_relative
|
4
|
-
require
|
2
|
+
APP_PATH = File.expand_path("../config/application", __dir__)
|
3
|
+
require_relative "../config/boot"
|
4
|
+
require "rails/commands"
|
data/spec/dummy/bin/rake
CHANGED
data/spec/dummy/bin/setup
CHANGED
@@ -1,38 +1,36 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
require
|
3
|
-
require 'fileutils'
|
4
|
-
include FileUtils
|
2
|
+
require "fileutils"
|
5
3
|
|
6
4
|
# path to your application root.
|
7
|
-
APP_ROOT =
|
5
|
+
APP_ROOT = File.expand_path("..", __dir__)
|
8
6
|
|
9
7
|
def system!(*args)
|
10
8
|
system(*args) || abort("\n== Command #{args} failed ==")
|
11
9
|
end
|
12
10
|
|
13
|
-
chdir APP_ROOT do
|
14
|
-
# This script is a
|
11
|
+
FileUtils.chdir APP_ROOT do
|
12
|
+
# This script is a way to set up or update your development environment automatically.
|
13
|
+
# This script is idempotent, so that you can run it at any time and get an expectable outcome.
|
15
14
|
# Add necessary setup steps to this file.
|
16
15
|
|
17
|
-
puts
|
18
|
-
system!
|
19
|
-
system(
|
20
|
-
|
21
|
-
# Install JavaScript dependencies if using Yarn
|
22
|
-
# system('bin/yarn')
|
16
|
+
puts "== Installing dependencies =="
|
17
|
+
system! "gem install bundler --conservative"
|
18
|
+
system("bundle check") || system!("bundle install")
|
23
19
|
|
20
|
+
# Install JavaScript dependencies
|
21
|
+
system! "bin/yarn"
|
24
22
|
|
25
23
|
# puts "\n== Copying sample files =="
|
26
24
|
# unless File.exist?('config/database.yml')
|
27
|
-
# cp 'config/database.yml.sample', 'config/database.yml'
|
25
|
+
# FileUtils.cp 'config/database.yml.sample', 'config/database.yml'
|
28
26
|
# end
|
29
27
|
|
30
28
|
puts "\n== Preparing database =="
|
31
|
-
system!
|
29
|
+
system! "bin/rails db:prepare"
|
32
30
|
|
33
31
|
puts "\n== Removing old logs and tempfiles =="
|
34
|
-
system!
|
32
|
+
system! "bin/rails log:clear tmp:clear"
|
35
33
|
|
36
34
|
puts "\n== Restarting application server =="
|
37
|
-
system!
|
35
|
+
system! "bin/rails restart"
|
38
36
|
end
|
data/spec/dummy/bin/update
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
require
|
3
|
-
require
|
4
|
-
include FileUtils
|
2
|
+
require "pathname"
|
3
|
+
require "fileutils"
|
4
|
+
include FileUtils # rubocop:disable Style/MixinUsage# rubocop:disable Style/MixinUsage
|
5
5
|
|
6
6
|
# path to your application root.
|
7
|
-
APP_ROOT = Pathname.new File.expand_path(
|
7
|
+
APP_ROOT = Pathname.new File.expand_path("../../", __FILE__)
|
8
8
|
|
9
9
|
def system!(*args)
|
10
10
|
system(*args) || abort("\n== Command #{args} failed ==")
|
@@ -14,16 +14,16 @@ chdir APP_ROOT do
|
|
14
14
|
# This script is a way to update your development environment automatically.
|
15
15
|
# Add necessary update steps to this file.
|
16
16
|
|
17
|
-
puts
|
18
|
-
system!
|
19
|
-
system(
|
17
|
+
puts "== Installing dependencies =="
|
18
|
+
system! "gem install bundler --conservative"
|
19
|
+
system("bundle check") || system!("bundle install")
|
20
20
|
|
21
21
|
puts "\n== Updating database =="
|
22
|
-
system!
|
22
|
+
system! "bin/rails db:migrate"
|
23
23
|
|
24
24
|
puts "\n== Removing old logs and tempfiles =="
|
25
|
-
system!
|
25
|
+
system! "bin/rails log:clear tmp:clear"
|
26
26
|
|
27
27
|
puts "\n== Restarting application server =="
|
28
|
-
system!
|
28
|
+
system! "bin/rails restart"
|
29
29
|
end
|
data/spec/dummy/bin/yarn
CHANGED
@@ -1,11 +1,17 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
Dir.chdir(
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
2
|
+
APP_ROOT = File.expand_path("..", __dir__)
|
3
|
+
Dir.chdir(APP_ROOT) do
|
4
|
+
yarn = ENV["PATH"].split(File::PATH_SEPARATOR)
|
5
|
+
.select { |dir| File.expand_path(dir) != __dir__ }
|
6
|
+
.product(["yarn", "yarn.cmd", "yarn.ps1"])
|
7
|
+
.map { |dir, file| File.expand_path(file, dir) }
|
8
|
+
.find { |file| File.executable?(file) }
|
9
|
+
|
10
|
+
if yarn
|
11
|
+
exec yarn, *ARGV
|
12
|
+
else
|
13
|
+
warn "Yarn executable was not detected in the system."
|
14
|
+
warn "Download Yarn at https://yarnpkg.com/en/docs/install"
|
9
15
|
exit 1
|
10
16
|
end
|
11
17
|
end
|
data/spec/dummy/config.ru
CHANGED
@@ -1,22 +1,27 @@
|
|
1
1
|
require_relative "boot"
|
2
2
|
|
3
3
|
require "rails/all"
|
4
|
+
require "nulogy_message_bus_producer"
|
4
5
|
|
6
|
+
# Require the gems listed in Gemfile, including any gems
|
7
|
+
# you've limited to :test, :development, or :production.
|
5
8
|
Bundler.require(*Rails.groups)
|
6
|
-
require "nulogy_message_bus_producer"
|
7
9
|
|
8
10
|
module Dummy
|
9
11
|
class Application < Rails::Application
|
10
12
|
# Initialize configuration defaults for originally generated Rails version.
|
11
13
|
config.load_defaults 5.1
|
12
14
|
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
config
|
15
|
+
# Configuration for the application, engines, and railties goes here.
|
16
|
+
#
|
17
|
+
# These settings can be overridden in specific environments using the files
|
18
|
+
# in config/environments, which are processed later.
|
19
|
+
#
|
20
|
+
# config.time_zone = "Central Time (US & Canada)"
|
21
|
+
# config.eager_load_paths << Rails.root.join("extras")
|
17
22
|
|
18
23
|
config.autoload_paths += %W[
|
19
|
-
#{config.root}/lib
|
24
|
+
"#{config.root}/lib"
|
20
25
|
]
|
21
26
|
end
|
22
27
|
end
|
data/spec/dummy/config/boot.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
|
2
|
-
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../Gemfile', __dir__)
|
1
|
+
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
|
3
2
|
|
4
|
-
require
|
5
|
-
$LOAD_PATH.unshift File.expand_path('../../../lib', __dir__)
|
3
|
+
require "bundler/setup" # Set up gems listed in the Gemfile.
|
data/spec/dummy/config/cable.yml
CHANGED
@@ -5,8 +5,8 @@ defaults: &defaults
|
|
5
5
|
adapter: postgresql
|
6
6
|
username: <%= ENV['NULOGY_MESSAGE_BUS_PRODUCER_DATABASE_USER'] || "nulogy" %>
|
7
7
|
password: <%= ENV['NULOGY_MESSAGE_BUS_PRODUCER_DATABASE_PASS'] || "Nulogy4Ever" %>
|
8
|
-
host: <%= ENV['
|
9
|
-
port:
|
8
|
+
host: <%= ENV['DATABASE_HOST'] || "localhost" %>
|
9
|
+
port: 5438
|
10
10
|
min_messages: warning
|
11
11
|
schema_search_path: "\"$user\", public"
|
12
12
|
|
@@ -1,8 +1,10 @@
|
|
1
|
+
require "active_support/core_ext/integer/time"
|
2
|
+
|
1
3
|
Rails.application.configure do
|
2
4
|
# Settings specified here will take precedence over those in config/application.rb.
|
3
5
|
|
4
|
-
# In the development environment your application's code is reloaded
|
5
|
-
#
|
6
|
+
# In the development environment your application's code is reloaded any time
|
7
|
+
# it changes. This slows down response time but is perfect for development
|
6
8
|
# since you don't have to restart the web server when you make code changes.
|
7
9
|
config.cache_classes = false
|
8
10
|
|
@@ -13,12 +15,14 @@ Rails.application.configure do
|
|
13
15
|
config.consider_all_requests_local = true
|
14
16
|
|
15
17
|
# Enable/disable caching. By default caching is disabled.
|
16
|
-
|
18
|
+
# Run rails dev:cache to toggle caching.
|
19
|
+
if Rails.root.join("tmp", "caching-dev.txt").exist?
|
17
20
|
config.action_controller.perform_caching = true
|
21
|
+
config.action_controller.enable_fragment_cache_logging = true
|
18
22
|
|
19
23
|
config.cache_store = :memory_store
|
20
24
|
config.public_file_server.headers = {
|
21
|
-
|
25
|
+
"Cache-Control" => "public, max-age=#{2.days.to_i}"
|
22
26
|
}
|
23
27
|
else
|
24
28
|
config.action_controller.perform_caching = false
|
@@ -26,6 +30,9 @@ Rails.application.configure do
|
|
26
30
|
config.cache_store = :null_store
|
27
31
|
end
|
28
32
|
|
33
|
+
# Store uploaded files on the local file system (see config/storage.yml for options).
|
34
|
+
config.active_storage.service = :local
|
35
|
+
|
29
36
|
# Don't care if the mailer can't send.
|
30
37
|
config.action_mailer.raise_delivery_errors = false
|
31
38
|
|
@@ -34,9 +41,18 @@ Rails.application.configure do
|
|
34
41
|
# Print deprecation notices to the Rails logger.
|
35
42
|
config.active_support.deprecation = :log
|
36
43
|
|
44
|
+
# Raise exceptions for disallowed deprecations.
|
45
|
+
config.active_support.disallowed_deprecation = :raise
|
46
|
+
|
47
|
+
# Tell Active Support which deprecation messages to disallow.
|
48
|
+
config.active_support.disallowed_deprecation_warnings = []
|
49
|
+
|
37
50
|
# Raise an error on page load if there are pending migrations.
|
38
51
|
config.active_record.migration_error = :page_load
|
39
52
|
|
53
|
+
# Highlight code that triggered database queries in logs.
|
54
|
+
config.active_record.verbose_query_logs = true
|
55
|
+
|
40
56
|
# Debug mode disables concatenation and preprocessing of assets.
|
41
57
|
# This option may cause significant delays in view rendering with a large
|
42
58
|
# number of complex assets.
|
@@ -45,10 +61,16 @@ Rails.application.configure do
|
|
45
61
|
# Suppress logger output for asset requests.
|
46
62
|
config.assets.quiet = true
|
47
63
|
|
48
|
-
# Raises error for missing translations
|
49
|
-
# config.
|
64
|
+
# Raises error for missing translations.
|
65
|
+
# config.i18n.raise_on_missing_translations = true
|
66
|
+
|
67
|
+
# Annotate rendered view with file names.
|
68
|
+
# config.action_view.annotate_rendered_view_with_filenames = true
|
50
69
|
|
51
70
|
# Use an evented file watcher to asynchronously detect changes in source code,
|
52
71
|
# routes, locales, etc. This feature depends on the listen gem.
|
53
|
-
|
72
|
+
config.file_watcher = ActiveSupport::EventedFileUpdateChecker
|
73
|
+
|
74
|
+
# Uncomment if you wish to allow Action Cable access from any origin.
|
75
|
+
# config.action_cable.disable_request_forgery_protection = true
|
54
76
|
end
|