saseo 0.1.0

data/lib/saseo/extensions/active_record/patcher.rb

@@ -0,0 +1,32 @@
+require 'saseo/extensions/active_record/detector'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+      module Patcher
+        extend self
+
+        UnsupportedActiveRecordVersion = Class.new(RuntimeError)
+
+        EXTENSIONS = {
+          3 => ['saseo/extensions/active_record/v_3'],
+          4 => ['saseo/extensions/active_record/v_4'],
+        }
+
+        def patch_active_record!
+          return false unless Saseo::Extensions::ActiveRecord::Detector.active_record_detected?
+          version = Saseo::Extensions::ActiveRecord::Detector.active_record_version
+
+          raise UnsupportedActiveRecordVersion.new unless EXTENSIONS.keys.include? version
+          EXTENSIONS[version].each do |version_extension|
+            require version_extension
+          end
+          true
+        end
+
+
+        patch_active_record!
+      end
+    end
+  end
+end

data/lib/saseo/extensions/active_record/v_3/connection_adapters/postgresql_adapter.rb

@@ -0,0 +1,26 @@
+require 'saseo/extensions/active_record/detector'
+
+require 'saseo/whodunnit'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+      module V3
+        module ConnectionAdapters
+          module PostgreSQLAdapter
+            extend self
+            def begin_db_transaction
+              execute "BEGIN"
+              Saseo::Whodunnit.set_db_whodunnit
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+if Saseo::Extensions::ActiveRecord::Detector.active_record_version == 3
+  require 'active_record/connection_adapters/postgresql_adapter'
+  ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend ::Saseo::Extensions::ActiveRecord::V3::ConnectionAdapters::PostgreSQLAdapter
+end

data/lib/saseo/extensions/active_record/v_3.rb

@@ -0,0 +1,11 @@
+require 'saseo/extensions/active_record/v_3/connection_adapters/postgresql_adapter'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+      module V3
+
+      end
+    end
+  end
+end

data/lib/saseo/extensions/active_record/v_4/connection_adapters/postgresql/database_statements.rb

@@ -0,0 +1,29 @@
+require 'saseo/extensions/active_record/detector'
+
+require 'saseo/whodunnit'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+      module V4
+        module ConnectionAdapters
+          module PostgreSQL
+            module DatabaseStatements
+              extend self
+
+              def begin_db_transaction
+                execute "BEGIN"
+                Saseo::Whodunnit.set_db_whodunnit
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+if Saseo::Extensions::ActiveRecord::Detector.active_record_version == 3
+  require 'active_record/connection_adapters/postgresql_adapter'
+  ::ActiveRecord::ConnectionAdapters::PostgreSQL::DatabaseStatements.prepend ::Saseo::Extensions::ActiveRecord::V4::ConnectionAdapters::PostgreSQL::DatabaseStatements
+end

data/lib/saseo/extensions/active_record/v_4.rb

@@ -0,0 +1,11 @@
+require 'saseo/extensions/active_record/v_4/connection_adapters/postgresql/database_statements'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+      module V4
+
+      end
+    end
+  end
+end

data/lib/saseo/extensions/active_record.rb

@@ -0,0 +1,9 @@
+require 'saseo/extensions/active_record/detector'
+require 'saseo/extensions/active_record/patcher'
+
+module Saseo
+  module Extensions
+    module ActiveRecord
+    end
+  end
+end

data/lib/saseo/extensions.rb

@@ -0,0 +1,7 @@
+require 'saseo/extensions/active_record'
+
+module Saseo
+  module Extensions
+
+  end
+end

data/lib/saseo/models/base.rb

@@ -0,0 +1,51 @@
+require 'active_record'
+require 'yaml'
+require 'saseo/config'
+
+module Saseo
+  module Models
+    class Base < ActiveRecord::Base
+      self.abstract_class = true
+      class << self
+        def database_config_path
+          Saseo.config.database_config_path
+        end
+
+        def database_config_url
+          Saseo.config.database_url
+        end
+
+        def database_config
+          database_config_url || database_config_from_file
+        end
+
+        def database_config_from_file
+          begin
+            database_config_from_relative_path
+          rescue
+            database_config_from_load_path
+          end
+        end
+
+        def database_config_from_relative_path
+          YAML::load(File.open(File.expand_path(database_config_path)))[Saseo.config.env]
+        end
+
+        def database_config_from_load_path
+          config = nil
+          config_path = database_config_path
+          $:.each do |load_path|
+            begin
+              config = YAML::load(File.open(File.expand_path File.join(load_path, config_path)))[Saseo.config.env]
+              break
+            rescue
+            end
+          end
+          config
+        end
+      end
+
+      establish_connection database_config
+    end
+  end
+end

data/lib/saseo/models/source/base.rb

@@ -0,0 +1,22 @@
+require 'saseo/models/base'
+
+module Saseo
+  module Models
+    module Source
+      class Base < Saseo::Models::Base
+        self.abstract_class = true
+        class << self
+          def database_config_path
+            Saseo.config.source_database_config_path
+          end
+
+          def database_config_url
+            Saseo.config.source_database_url
+          end
+        end
+
+        establish_connection database_config
+      end
+    end
+  end
+end

data/lib/saseo/models/source/version.rb

@@ -0,0 +1,20 @@
+require 'saseo/models/source/base'
+require 'saseo/config'
+
+module Saseo
+  module Models
+    module Source
+      class Version < Saseo::Models::Source::Base
+        self.table_name = Saseo.config.source_table_name
+
+        validates :transaction_id, presence: true
+        validates :table_name, presence: true
+        validates :action_timestamp, presence: true
+        validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
+        validates :whodunnit, presence: true
+        validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
+        validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
+      end
+    end
+  end
+end

data/lib/saseo/models/version.rb

@@ -0,0 +1,19 @@
+require 'saseo/models/base'
+require 'saseo/config'
+
+module Saseo
+  module Models
+    class Version < Saseo::Models::Base
+      self.table_name = Saseo.config.table_name
+
+      validates :transaction_id, presence: true
+      validates :table_name, presence: true
+      validates :item_id, presence: true
+      validates :action_timestamp, presence: true
+      validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
+      validates :whodunnit, presence: true
+      validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
+      validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
+    end
+  end
+end

data/lib/saseo/persistence/consumer.rb

@@ -0,0 +1,25 @@
+require 'philotic/consumer'
+require 'saseo/config'
+require 'saseo/persistence/persistor'
+
+module Saseo
+  module Persistence
+    class Consumer < Philotic::Consumer
+      # subscribe_to philotic_message_type: :'saseo.record_audit'
+      subscribe_to Saseo.config.consumer_philotic_subscription
+
+      manually_acknowledge
+
+      def consume(message)
+        begin
+          Saseo::Persistence::Persistor.persist! message
+
+          acknowledge message
+        rescue => e
+          # be sure to create a queue that monitors for saseo_consumer_error: true
+          Philotic::Message.publish({saseo_consumer_error: true, message: e.message, error_class: e.class.name, trace: e.backtrace})
+        end
+      end
+    end
+  end
+end

data/lib/saseo/persistence/persistor.rb

@@ -0,0 +1,37 @@
+require 'oj'
+require 'saseo/models/version'
+
+module Saseo
+  module Persistence
+    module Persistor
+      extend self
+
+      def version_attributes
+        %i[
+            id
+            transaction_id
+            table_name
+            item_id
+            item_uuid
+            action_timestamp
+            action
+            whodunnit
+          ]
+      end
+
+      def persist!(message)
+        version_record = Saseo::Models::Version.new
+
+        version_attributes.each do |attr|
+          version_record.send("#{attr}=", message.send(attr))
+        end
+
+        # ActiveRecord 3 doesn't handle jsonb columns properly
+        version_record.old_data = message.old_data && Oj.dump(message.old_data)
+        version_record.new_data = message.new_data && Oj.dump(message.new_data)
+
+        version_record.save!
+      end
+    end
+  end
+end

data/lib/saseo/persistence.rb

@@ -0,0 +1,8 @@
+require 'saseo/persistence/consumer'
+require 'saseo/persistence/persistor'
+
+module Saseo
+  module Persistence
+
+  end
+end

data/lib/saseo/publishing/data_change_message.rb

@@ -0,0 +1,43 @@
+require 'philotic/message'
+require 'oj'
+
+module Saseo
+  module Publishing
+    class DataChangeMessage < Philotic::Message
+
+      COMPONENT    = :saseo
+      MESSAGE_TYPE = :'saseo.record_audit'
+
+      attr_routable :table_name, :action, :whodunnit, :item_id, :item_uuid, :transaction_id
+      attr_payload :id, :old_data, :new_data, :action_timestamp
+
+      def initialize(version)
+
+        super({})
+        @id               = version.id
+        @transaction_id   = version.transaction_id
+        @table_name       = version.table_name
+        @action           = version.action
+        @whodunnit        = version.whodunnit
+        @action_timestamp = version.action_timestamp
+
+        # ActiveRecord 3 doesn't handle jsonb columns properly
+        @old_data         = ensure_json_load version.old_data
+        @new_data         = ensure_json_load version.new_data
+
+        @item_id   = @new_data ? @new_data['id'] : @old_data['id']
+        @item_uuid = @new_data ? @new_data['uuid'] : @old_data['uuid']
+
+        @philotic_component    = COMPONENT
+        @philotic_message_type = MESSAGE_TYPE
+
+      end
+
+
+      def ensure_json_load(val)
+        return val unless val.is_a? String
+        Oj.load val
+      end
+    end
+  end
+end

data/lib/saseo/publishing/publisher.rb

@@ -0,0 +1,95 @@
+require 'active_record'
+require 'saseo/config'
+require 'saseo/publishing/data_change_message'
+require 'saseo/models/source/version'
+require 'logger'
+
+module Saseo
+  module Publishing
+    class Publisher
+
+      attr_reader :channels
+      attr_writer :logger
+
+      def initialize(channels: nil)
+        self.channels = channels
+      end
+
+      def logger
+        @logger ||= Logger.new(STDOUT)
+      end
+
+      def channels=(val)
+        if val == :all
+          tables = Saseo::Models::Source::Base.connection.tables
+
+          @channels = tables.reduce([]) do |channels, table|
+            %w[insert update delete].each do |operation|
+              channels << "SASEO_#{table}_#{operation}_AUDIT".upcase
+            end
+            channels
+          end
+        else
+          @channels = Array(val).flatten
+        end
+      end
+
+      def run(channels: :all, loop: true, timeout: nil)
+        self.channels = channels
+        logger.info "listening on #{self.channels}..."
+
+        listen channels: self.channels, loop: loop, timeout: timeout do |channel, pid, payload|
+          uuid = payload
+          Saseo::Models::Source::Base.transaction do
+
+            record = Saseo::Models::Source::Version.find(uuid)
+            Saseo::Publishing::DataChangeMessage.publish(record)
+            record.delete
+          end
+        end
+      end
+
+      def notify(channel, payload)
+        conn = Saseo::Models::Source::Base.connection.instance_variable_get(:@connection)
+        conn.async_exec "SELECT pg_notify('#{channel}', '#{payload.to_s}')"
+      end
+
+      # Heavily cribbed from Sequel:
+      # https://github.com/jeremyevans/sequel/blob/c6678741ce34aac52cff966ff6a6288ccb8d5b75/lib/sequel/adapters/postgres.rb#L440
+      def listen(channels:, after_listen: nil, timeout: nil, loop: false, &block)
+        if loop && !block
+          raise ArgumentError, 'calling #listen with :loop requires a block'
+        end
+        channels = Array(channels)
+
+        timeout_block = timeout.respond_to?(:call) ? timeout : proc { timeout }
+        loop_callable = loop.respond_to?(:call)
+        Saseo::Models::Source::Base.connection_pool.with_connection do |connection|
+          begin
+            conn = connection.instance_variable_get(:@connection)
+            channels.each do |channel|
+              conn.async_exec "LISTEN \"#{channel}\""
+              logger.debug { "listening to #{channel}..." }
+
+            end
+            after_listen.call(conn) if after_listen
+
+            if loop
+
+              catch(:stop) do
+                loop do
+                  conn.wait_for_notify(timeout_block.call, &block)
+                  loop.call(conn) if loop_callable
+                end
+              end
+            else
+              conn.wait_for_notify(timeout_block.call, &block)
+            end
+          ensure
+            conn.async_exec 'UNLISTEN *'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saseo/publishing.rb

@@ -0,0 +1,8 @@
+require 'saseo/publishing/data_change_message'
+require 'saseo/publishing/publisher'
+
+module Saseo
+  module Publishing
+
+  end
+end

data/lib/saseo/version.rb

@@ -0,0 +1,3 @@
+module Saseo
+  VERSION = "0.1.0"
+end

data/lib/saseo/whodunnit.rb

@@ -0,0 +1,41 @@
+require 'saseo/extensions/active_record/detector'
+
+module Saseo
+
+  module Whodunnit
+    extend self
+
+    def whodunnit
+      @@whodunnit ||= nil
+    end
+
+    def whodunnit=(val)
+      @@whodunnit = val
+      if Saseo::Extensions::ActiveRecord::Detector.active_record_detected? && ActiveRecord::Base.connected?
+        set_db_whodunnit(whodunnit)
+      end
+    end
+
+    def impersonate(who)
+      whodidit    = whodunnit
+      @@whodunnit = who
+      yield
+    ensure
+      @@whodunnit = whodidit
+    end
+
+    def set_db_whodunnit(who = nil)
+      who ||= whodunnit
+      connection.execute "SET saseo.whodunnit TO '#{connection.quote_string who.to_s}'"
+    end
+
+    def connection
+      return ActiveRecord::Base.connection if Saseo::Extensions::ActiveRecord::Detector.active_record_detected? && ActiveRecord::Base.connected?
+    end
+  end
+
+  class << self
+    extend Forwardable
+    def_delegators Saseo::Whodunnit, :whodunnit, :whodunnit=, :impersonate
+  end
+end

data/lib/saseo.rb

@@ -0,0 +1,10 @@
+require "saseo/version"
+# require all extensions here
+require 'saseo/config'
+require 'saseo/extensions'
+require 'saseo/publishing'
+require 'saseo/whodunnit'
+
+module Saseo
+
+end

data/saseo.example.yml

@@ -0,0 +1,12 @@
+defaults: &defaults
+  channels:
+  - SASEO_AUDIT
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults
+
+production:
+  <<: *defaults

data/saseo.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'saseo/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'saseo'
+  spec.version       = Saseo::VERSION
+  spec.authors       = ['Nathan Keyes']
+  spec.email         = ['nathan.keyes@avantcredit.com']
+
+  spec.summary       = %q{RabbitMQ based replacement for PaperTrail}
+  spec.description   = %q{RabbitMQ based replacement for PaperTrail}
+  spec.homepage      = 'https://github.com/avantcredit/saseo'
+
+  spec.files         = `git ls-files`.split($\).reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'awesome_print'
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'codeclimate-test-reporter', '~> 0.4'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'generator_spec'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+
+  spec.add_dependency 'oj'
+  spec.add_dependency 'pg'
+  spec.add_dependency 'philotic', '~> 1.2'
+  spec.add_dependency 'rails', '>= 3.2'
+  spec.add_dependency 'standalone_migrations', '>= 2.1.5'
+end

data/tasks/bump.rake

@@ -0,0 +1,30 @@
+desc 'shortcut for bump:patch'
+task bump: 'bump:patch'
+
+namespace :bump do
+  desc 'Bump x.y.Z'
+  task :patch
+
+  desc 'Bump x.Y.z'
+  task :minor
+
+  desc 'Bump X.y.z'
+  task :major
+end
+
+# extracted and modified from https://github.com/grosser/project_template
+rule /^bump:.*/ do |t|
+  sh "git status | grep 'nothing to commit'" # ensure we are not dirty
+  index = ['major', 'minor', 'patch'].index(t.name.split(':').last)
+  file  = 'lib/saseo/version.rb'
+
+  version_file                = File.read(file)
+  old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
+  version_parts[index]        = version_parts[index].to_i + 1
+  version_parts[2]            = 0 if index < 2
+  version_parts[1]            = 0 if index < 1
+  new_version                 = version_parts * '.'
+  File.open(file, 'w') { |f| f.write(version_file.sub(old_version, new_version)) }
+
+  sh "bundle && git add #{file} && git commit -m 'bump version to #{new_version}'"
+end