saseo 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/.rspec +2 -0
  4. data/.ruby-gemset +1 -0
  5. data/.ruby-version +1 -0
  6. data/.saseo.standalone_migrations +6 -0
  7. data/.saseo_source.standalone_migrations +6 -0
  8. data/.travis.yml +38 -0
  9. data/Gemfile +4 -0
  10. data/LICENSE +22 -0
  11. data/README.md +38 -0
  12. data/Rakefile +12 -0
  13. data/bin/console +14 -0
  14. data/bin/setup +7 -0
  15. data/db/saseo/migrate/20151028181502_initial_schema.rb +18 -0
  16. data/db/saseo/schema.rb +32 -0
  17. data/db/saseo_source/migrate/20151028181502_initial_schema.rb +16 -0
  18. data/db/saseo_source/schema.rb +30 -0
  19. data/exe/saseo_consumer +6 -0
  20. data/exe/saseo_publisher +6 -0
  21. data/lib/generators/saseo/install_generator.rb +35 -0
  22. data/lib/generators/saseo/templates/add_saseo_trigger.rb.erb +14 -0
  23. data/lib/generators/saseo/templates/add_saseo_trigger_function.rb +105 -0
  24. data/lib/generators/saseo/trigger_generator.rb +42 -0
  25. data/lib/saseo/config/defaults.rb +41 -0
  26. data/lib/saseo/config/saseo_database.yml +14 -0
  27. data/lib/saseo/config/saseo_source_database.yml +14 -0
  28. data/lib/saseo/config.rb +52 -0
  29. data/lib/saseo/extensions/active_record/detector.rb +17 -0
  30. data/lib/saseo/extensions/active_record/patcher.rb +32 -0
  31. data/lib/saseo/extensions/active_record/v_3/connection_adapters/postgresql_adapter.rb +26 -0
  32. data/lib/saseo/extensions/active_record/v_3.rb +11 -0
  33. data/lib/saseo/extensions/active_record/v_4/connection_adapters/postgresql/database_statements.rb +29 -0
  34. data/lib/saseo/extensions/active_record/v_4.rb +11 -0
  35. data/lib/saseo/extensions/active_record.rb +9 -0
  36. data/lib/saseo/extensions.rb +7 -0
  37. data/lib/saseo/models/base.rb +51 -0
  38. data/lib/saseo/models/source/base.rb +22 -0
  39. data/lib/saseo/models/source/version.rb +20 -0
  40. data/lib/saseo/models/version.rb +19 -0
  41. data/lib/saseo/persistence/consumer.rb +25 -0
  42. data/lib/saseo/persistence/persistor.rb +37 -0
  43. data/lib/saseo/persistence.rb +8 -0
  44. data/lib/saseo/publishing/data_change_message.rb +43 -0
  45. data/lib/saseo/publishing/publisher.rb +95 -0
  46. data/lib/saseo/publishing.rb +8 -0
  47. data/lib/saseo/version.rb +3 -0
  48. data/lib/saseo/whodunnit.rb +41 -0
  49. data/lib/saseo.rb +10 -0
  50. data/saseo.example.yml +12 -0
  51. data/saseo.gemspec +34 -0
  52. data/tasks/bump.rake +30 -0
  53. metadata +264 -0
@@ -0,0 +1,32 @@
1
+ require 'saseo/extensions/active_record/detector'
2
+
3
+ module Saseo
4
+ module Extensions
5
+ module ActiveRecord
6
+ module Patcher
7
+ extend self
8
+
9
+ UnsupportedActiveRecordVersion = Class.new(RuntimeError)
10
+
11
+ EXTENSIONS = {
12
+ 3 => ['saseo/extensions/active_record/v_3'],
13
+ 4 => ['saseo/extensions/active_record/v_4'],
14
+ }
15
+
16
+ def patch_active_record!
17
+ return false unless Saseo::Extensions::ActiveRecord::Detector.active_record_detected?
18
+ version = Saseo::Extensions::ActiveRecord::Detector.active_record_version
19
+
20
+ raise UnsupportedActiveRecordVersion.new unless EXTENSIONS.keys.include? version
21
+ EXTENSIONS[version].each do |version_extension|
22
+ require version_extension
23
+ end
24
+ true
25
+ end
26
+
27
+
28
+ patch_active_record!
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,26 @@
1
+ require 'saseo/extensions/active_record/detector'
2
+
3
+ require 'saseo/whodunnit'
4
+
5
+ module Saseo
6
+ module Extensions
7
+ module ActiveRecord
8
+ module V3
9
+ module ConnectionAdapters
10
+ module PostgreSQLAdapter
11
+ extend self
12
+ def begin_db_transaction
13
+ execute "BEGIN"
14
+ Saseo::Whodunnit.set_db_whodunnit
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+
23
+ if Saseo::Extensions::ActiveRecord::Detector.active_record_version == 3
24
+ require 'active_record/connection_adapters/postgresql_adapter'
25
+ ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend ::Saseo::Extensions::ActiveRecord::V3::ConnectionAdapters::PostgreSQLAdapter
26
+ end
@@ -0,0 +1,11 @@
1
+ require 'saseo/extensions/active_record/v_3/connection_adapters/postgresql_adapter'
2
+
3
+ module Saseo
4
+ module Extensions
5
+ module ActiveRecord
6
+ module V3
7
+
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,29 @@
1
+ require 'saseo/extensions/active_record/detector'
2
+
3
+ require 'saseo/whodunnit'
4
+
5
+ module Saseo
6
+ module Extensions
7
+ module ActiveRecord
8
+ module V4
9
+ module ConnectionAdapters
10
+ module PostgreSQL
11
+ module DatabaseStatements
12
+ extend self
13
+
14
+ def begin_db_transaction
15
+ execute "BEGIN"
16
+ Saseo::Whodunnit.set_db_whodunnit
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ if Saseo::Extensions::ActiveRecord::Detector.active_record_version == 3
27
+ require 'active_record/connection_adapters/postgresql_adapter'
28
+ ::ActiveRecord::ConnectionAdapters::PostgreSQL::DatabaseStatements.prepend ::Saseo::Extensions::ActiveRecord::V4::ConnectionAdapters::PostgreSQL::DatabaseStatements
29
+ end
@@ -0,0 +1,11 @@
1
+ require 'saseo/extensions/active_record/v_4/connection_adapters/postgresql/database_statements'
2
+
3
+ module Saseo
4
+ module Extensions
5
+ module ActiveRecord
6
+ module V4
7
+
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,9 @@
1
+ require 'saseo/extensions/active_record/detector'
2
+ require 'saseo/extensions/active_record/patcher'
3
+
4
+ module Saseo
5
+ module Extensions
6
+ module ActiveRecord
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,7 @@
1
+ require 'saseo/extensions/active_record'
2
+
3
+ module Saseo
4
+ module Extensions
5
+
6
+ end
7
+ end
@@ -0,0 +1,51 @@
1
+ require 'active_record'
2
+ require 'yaml'
3
+ require 'saseo/config'
4
+
5
+ module Saseo
6
+ module Models
7
+ class Base < ActiveRecord::Base
8
+ self.abstract_class = true
9
+ class << self
10
+ def database_config_path
11
+ Saseo.config.database_config_path
12
+ end
13
+
14
+ def database_config_url
15
+ Saseo.config.database_url
16
+ end
17
+
18
+ def database_config
19
+ database_config_url || database_config_from_file
20
+ end
21
+
22
+ def database_config_from_file
23
+ begin
24
+ database_config_from_relative_path
25
+ rescue
26
+ database_config_from_load_path
27
+ end
28
+ end
29
+
30
+ def database_config_from_relative_path
31
+ YAML::load(File.open(File.expand_path(database_config_path)))[Saseo.config.env]
32
+ end
33
+
34
+ def database_config_from_load_path
35
+ config = nil
36
+ config_path = database_config_path
37
+ $:.each do |load_path|
38
+ begin
39
+ config = YAML::load(File.open(File.expand_path File.join(load_path, config_path)))[Saseo.config.env]
40
+ break
41
+ rescue
42
+ end
43
+ end
44
+ config
45
+ end
46
+ end
47
+
48
+ establish_connection database_config
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,22 @@
1
+ require 'saseo/models/base'
2
+
3
+ module Saseo
4
+ module Models
5
+ module Source
6
+ class Base < Saseo::Models::Base
7
+ self.abstract_class = true
8
+ class << self
9
+ def database_config_path
10
+ Saseo.config.source_database_config_path
11
+ end
12
+
13
+ def database_config_url
14
+ Saseo.config.source_database_url
15
+ end
16
+ end
17
+
18
+ establish_connection database_config
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,20 @@
1
+ require 'saseo/models/source/base'
2
+ require 'saseo/config'
3
+
4
+ module Saseo
5
+ module Models
6
+ module Source
7
+ class Version < Saseo::Models::Source::Base
8
+ self.table_name = Saseo.config.source_table_name
9
+
10
+ validates :transaction_id, presence: true
11
+ validates :table_name, presence: true
12
+ validates :action_timestamp, presence: true
13
+ validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
14
+ validates :whodunnit, presence: true
15
+ validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
16
+ validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,19 @@
1
+ require 'saseo/models/base'
2
+ require 'saseo/config'
3
+
4
+ module Saseo
5
+ module Models
6
+ class Version < Saseo::Models::Base
7
+ self.table_name = Saseo.config.table_name
8
+
9
+ validates :transaction_id, presence: true
10
+ validates :table_name, presence: true
11
+ validates :item_id, presence: true
12
+ validates :action_timestamp, presence: true
13
+ validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
14
+ validates :whodunnit, presence: true
15
+ validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
16
+ validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,25 @@
1
+ require 'philotic/consumer'
2
+ require 'saseo/config'
3
+ require 'saseo/persistence/persistor'
4
+
5
+ module Saseo
6
+ module Persistence
7
+ class Consumer < Philotic::Consumer
8
+ # subscribe_to philotic_message_type: :'saseo.record_audit'
9
+ subscribe_to Saseo.config.consumer_philotic_subscription
10
+
11
+ manually_acknowledge
12
+
13
+ def consume(message)
14
+ begin
15
+ Saseo::Persistence::Persistor.persist! message
16
+
17
+ acknowledge message
18
+ rescue => e
19
+ # be sure to create a queue that monitors for saseo_consumer_error: true
20
+ Philotic::Message.publish({saseo_consumer_error: true, message: e.message, error_class: e.class.name, trace: e.backtrace})
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,37 @@
1
+ require 'oj'
2
+ require 'saseo/models/version'
3
+
4
+ module Saseo
5
+ module Persistence
6
+ module Persistor
7
+ extend self
8
+
9
+ def version_attributes
10
+ %i[
11
+ id
12
+ transaction_id
13
+ table_name
14
+ item_id
15
+ item_uuid
16
+ action_timestamp
17
+ action
18
+ whodunnit
19
+ ]
20
+ end
21
+
22
+ def persist!(message)
23
+ version_record = Saseo::Models::Version.new
24
+
25
+ version_attributes.each do |attr|
26
+ version_record.send("#{attr}=", message.send(attr))
27
+ end
28
+
29
+ # ActiveRecord 3 doesn't handle jsonb columns properly
30
+ version_record.old_data = message.old_data && Oj.dump(message.old_data)
31
+ version_record.new_data = message.new_data && Oj.dump(message.new_data)
32
+
33
+ version_record.save!
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,8 @@
1
+ require 'saseo/persistence/consumer'
2
+ require 'saseo/persistence/persistor'
3
+
4
+ module Saseo
5
+ module Persistence
6
+
7
+ end
8
+ end
@@ -0,0 +1,43 @@
1
+ require 'philotic/message'
2
+ require 'oj'
3
+
4
+ module Saseo
5
+ module Publishing
6
+ class DataChangeMessage < Philotic::Message
7
+
8
+ COMPONENT = :saseo
9
+ MESSAGE_TYPE = :'saseo.record_audit'
10
+
11
+ attr_routable :table_name, :action, :whodunnit, :item_id, :item_uuid, :transaction_id
12
+ attr_payload :id, :old_data, :new_data, :action_timestamp
13
+
14
+ def initialize(version)
15
+
16
+ super({})
17
+ @id = version.id
18
+ @transaction_id = version.transaction_id
19
+ @table_name = version.table_name
20
+ @action = version.action
21
+ @whodunnit = version.whodunnit
22
+ @action_timestamp = version.action_timestamp
23
+
24
+ # ActiveRecord 3 doesn't handle jsonb columns properly
25
+ @old_data = ensure_json_load version.old_data
26
+ @new_data = ensure_json_load version.new_data
27
+
28
+ @item_id = @new_data ? @new_data['id'] : @old_data['id']
29
+ @item_uuid = @new_data ? @new_data['uuid'] : @old_data['uuid']
30
+
31
+ @philotic_component = COMPONENT
32
+ @philotic_message_type = MESSAGE_TYPE
33
+
34
+ end
35
+
36
+
37
+ def ensure_json_load(val)
38
+ return val unless val.is_a? String
39
+ Oj.load val
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,95 @@
1
+ require 'active_record'
2
+ require 'saseo/config'
3
+ require 'saseo/publishing/data_change_message'
4
+ require 'saseo/models/source/version'
5
+ require 'logger'
6
+
7
+ module Saseo
8
+ module Publishing
9
+ class Publisher
10
+
11
+ attr_reader :channels
12
+ attr_writer :logger
13
+
14
+ def initialize(channels: nil)
15
+ self.channels = channels
16
+ end
17
+
18
+ def logger
19
+ @logger ||= Logger.new(STDOUT)
20
+ end
21
+
22
+ def channels=(val)
23
+ if val == :all
24
+ tables = Saseo::Models::Source::Base.connection.tables
25
+
26
+ @channels = tables.reduce([]) do |channels, table|
27
+ %w[insert update delete].each do |operation|
28
+ channels << "SASEO_#{table}_#{operation}_AUDIT".upcase
29
+ end
30
+ channels
31
+ end
32
+ else
33
+ @channels = Array(val).flatten
34
+ end
35
+ end
36
+
37
+ def run(channels: :all, loop: true, timeout: nil)
38
+ self.channels = channels
39
+ logger.info "listening on #{self.channels}..."
40
+
41
+ listen channels: self.channels, loop: loop, timeout: timeout do |channel, pid, payload|
42
+ uuid = payload
43
+ Saseo::Models::Source::Base.transaction do
44
+
45
+ record = Saseo::Models::Source::Version.find(uuid)
46
+ Saseo::Publishing::DataChangeMessage.publish(record)
47
+ record.delete
48
+ end
49
+ end
50
+ end
51
+
52
+ def notify(channel, payload)
53
+ conn = Saseo::Models::Source::Base.connection.instance_variable_get(:@connection)
54
+ conn.async_exec "SELECT pg_notify('#{channel}', '#{payload.to_s}')"
55
+ end
56
+
57
+ # Heavily cribbed from Sequel:
58
+ # https://github.com/jeremyevans/sequel/blob/c6678741ce34aac52cff966ff6a6288ccb8d5b75/lib/sequel/adapters/postgres.rb#L440
59
+ def listen(channels:, after_listen: nil, timeout: nil, loop: false, &block)
60
+ if loop && !block
61
+ raise ArgumentError, 'calling #listen with :loop requires a block'
62
+ end
63
+ channels = Array(channels)
64
+
65
+ timeout_block = timeout.respond_to?(:call) ? timeout : proc { timeout }
66
+ loop_callable = loop.respond_to?(:call)
67
+ Saseo::Models::Source::Base.connection_pool.with_connection do |connection|
68
+ begin
69
+ conn = connection.instance_variable_get(:@connection)
70
+ channels.each do |channel|
71
+ conn.async_exec "LISTEN \"#{channel}\""
72
+ logger.debug { "listening to #{channel}..." }
73
+
74
+ end
75
+ after_listen.call(conn) if after_listen
76
+
77
+ if loop
78
+
79
+ catch(:stop) do
80
+ loop do
81
+ conn.wait_for_notify(timeout_block.call, &block)
82
+ loop.call(conn) if loop_callable
83
+ end
84
+ end
85
+ else
86
+ conn.wait_for_notify(timeout_block.call, &block)
87
+ end
88
+ ensure
89
+ conn.async_exec 'UNLISTEN *'
90
+ end
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,8 @@
1
+ require 'saseo/publishing/data_change_message'
2
+ require 'saseo/publishing/publisher'
3
+
4
+ module Saseo
5
+ module Publishing
6
+
7
+ end
8
+ end
@@ -0,0 +1,3 @@
1
+ module Saseo
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,41 @@
1
+ require 'saseo/extensions/active_record/detector'
2
+
3
+ module Saseo
4
+
5
+ module Whodunnit
6
+ extend self
7
+
8
+ def whodunnit
9
+ @@whodunnit ||= nil
10
+ end
11
+
12
+ def whodunnit=(val)
13
+ @@whodunnit = val
14
+ if Saseo::Extensions::ActiveRecord::Detector.active_record_detected? && ActiveRecord::Base.connected?
15
+ set_db_whodunnit(whodunnit)
16
+ end
17
+ end
18
+
19
+ def impersonate(who)
20
+ whodidit = whodunnit
21
+ @@whodunnit = who
22
+ yield
23
+ ensure
24
+ @@whodunnit = whodidit
25
+ end
26
+
27
+ def set_db_whodunnit(who = nil)
28
+ who ||= whodunnit
29
+ connection.execute "SET saseo.whodunnit TO '#{connection.quote_string who.to_s}'"
30
+ end
31
+
32
+ def connection
33
+ return ActiveRecord::Base.connection if Saseo::Extensions::ActiveRecord::Detector.active_record_detected? && ActiveRecord::Base.connected?
34
+ end
35
+ end
36
+
37
+ class << self
38
+ extend Forwardable
39
+ def_delegators Saseo::Whodunnit, :whodunnit, :whodunnit=, :impersonate
40
+ end
41
+ end
data/lib/saseo.rb ADDED
@@ -0,0 +1,10 @@
1
+ require "saseo/version"
2
+ # require all extensions here
3
+ require 'saseo/config'
4
+ require 'saseo/extensions'
5
+ require 'saseo/publishing'
6
+ require 'saseo/whodunnit'
7
+
8
+ module Saseo
9
+
10
+ end
data/saseo.example.yml ADDED
@@ -0,0 +1,12 @@
1
+ defaults: &defaults
2
+ channels:
3
+ - SASEO_AUDIT
4
+
5
+ development:
6
+ <<: *defaults
7
+
8
+ test:
9
+ <<: *defaults
10
+
11
+ production:
12
+ <<: *defaults
data/saseo.gemspec ADDED
@@ -0,0 +1,34 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'saseo/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'saseo'
8
+ spec.version = Saseo::VERSION
9
+ spec.authors = ['Nathan Keyes']
10
+ spec.email = ['nathan.keyes@avantcredit.com']
11
+
12
+ spec.summary = %q{RabbitMQ based replacement for PaperTrail}
13
+ spec.description = %q{RabbitMQ based replacement for PaperTrail}
14
+ spec.homepage = 'https://github.com/avantcredit/saseo'
15
+
16
+ spec.files = `git ls-files`.split($\).reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = 'exe'
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_development_dependency 'awesome_print'
22
+ spec.add_development_dependency 'bundler', '~> 1.10'
23
+ spec.add_development_dependency 'codeclimate-test-reporter', '~> 0.4'
24
+ spec.add_development_dependency 'database_cleaner'
25
+ spec.add_development_dependency 'generator_spec'
26
+ spec.add_development_dependency 'rake', '~> 10.0'
27
+ spec.add_development_dependency 'rspec'
28
+
29
+ spec.add_dependency 'oj'
30
+ spec.add_dependency 'pg'
31
+ spec.add_dependency 'philotic', '~> 1.2'
32
+ spec.add_dependency 'rails', '>= 3.2'
33
+ spec.add_dependency 'standalone_migrations', '>= 2.1.5'
34
+ end
data/tasks/bump.rake ADDED
@@ -0,0 +1,30 @@
1
+ desc 'shortcut for bump:patch'
2
+ task bump: 'bump:patch'
3
+
4
+ namespace :bump do
5
+ desc 'Bump x.y.Z'
6
+ task :patch
7
+
8
+ desc 'Bump x.Y.z'
9
+ task :minor
10
+
11
+ desc 'Bump X.y.z'
12
+ task :major
13
+ end
14
+
15
+ # extracted and modified from https://github.com/grosser/project_template
16
+ rule /^bump:.*/ do |t|
17
+ sh "git status | grep 'nothing to commit'" # ensure we are not dirty
18
+ index = ['major', 'minor', 'patch'].index(t.name.split(':').last)
19
+ file = 'lib/saseo/version.rb'
20
+
21
+ version_file = File.read(file)
22
+ old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
23
+ version_parts[index] = version_parts[index].to_i + 1
24
+ version_parts[2] = 0 if index < 2
25
+ version_parts[1] = 0 if index < 1
26
+ new_version = version_parts * '.'
27
+ File.open(file, 'w') { |f| f.write(version_file.sub(old_version, new_version)) }
28
+
29
+ sh "bundle && git add #{file} && git commit -m 'bump version to #{new_version}'"
30
+ end