saseo 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 365c9551971dcd5e511dd8bfceffbf60ead36c2f
-  data.tar.gz: 3cee7c15a02815d3c9fea3c92d6d85d61c21cd7c
+  metadata.gz: b329bf7dd271331dda62ef081a05d9f7e0c64af9
+  data.tar.gz: ed09bc9155ee23828beee19b3fa32e0c781307e5
 SHA512:
-  metadata.gz: 78ffd353c6977bba243eceed9794711449d62a0df65b593bbf25005eb41da2cde0e591f4ff65e070628dba305d2b0d9dbb7135ec111af721156b80a402bffc7a
-  data.tar.gz: e3f4d6356a9c3b117bc9e1eccc301e1456a1035612c2e5f590f1cba8fe2493ece7ef28fde3ee577854f0dcb67ab56f0a88bd0f1faca4af32dd18b7d94a96b1af
+  metadata.gz: 993d46fb9c8d728324a334c627bdfefae21d53668d9c431bc615bd343370d04ea3c0636e83ecb1cc15d6014232498acaf5e26765c4e3410b67334e670e5ace9f
+  data.tar.gz: 8c270a1382502571d728940c10509d00f5f8a18c4d0bcb181b11088fe98180d08d259d112b3ebbab4be463491b6ac9ae526235e846b814ac34bc97a9f23db8ce

data/README.md

@@ -5,6 +5,9 @@ RabbitMQ based PaperTrail replacement
 [![Dependency Status](https://gemnasium.com/avantcredit/saseo.svg)](https://gemnasium.com/avantcredit/saseo)
 [![Code Climate](https://codeclimate.com/github/avantcredit/saseo/badges/gpa.svg)](https://codeclimate.com/github/avantcredit/saseo)
 [![Test Coverage](https://codeclimate.com/github/avantcredit/saseo/badges/coverage.svg)](https://codeclimate.com/github/avantcredit/saseo)
+[![Gem Version](http://img.shields.io/gem/v/saseo.svg)](https://rubygems.org/gems/saseo)
+[![License](http://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org)
+
 
 ## Installation
 
@@ -26,6 +29,8 @@ Or install it yourself as:
 
 TODO: Write usage instructions here
 
+## Sample Setup
+![](https://raw.githubusercontent.com/avantcredit/saseo/master/sample_saseo_setup.png)
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/saseo/config.rb

@@ -30,10 +30,6 @@ module Saseo
       load_config(config[env])
     end
 
-    def channels
-      @channels ||= _load_json_config :channels
-    end
-
     def consumer_philotic_subscription
       @consumer_philotic_subscription ||= _load_json_config :consumer_philotic_subscription
     end

data/lib/saseo/config/defaults.rb

@@ -11,6 +11,7 @@ module Saseo
       SOURCE_TABLE_SCHEMA            = 'saseo'
       SOURCE_TABLE_NAME              = 'saseo_source_versions'
       CONSUMER_PHILOTIC_SUBSCRIPTION = 'saseo_audit'
+      IGNORE_FIELDS_CONFIG_PATH      = nil
 
       def defaults
         @defaults ||= Hash[Saseo::Config::Defaults.constants.map do |c|

data/lib/saseo/models/base.rb

@@ -32,7 +32,7 @@ module Saseo
         end
 
         def database_config_from_load_path
-          config = nil
+          config      = nil
           config_path = database_config_path
           $:.each do |load_path|
             begin
@@ -43,9 +43,44 @@ module Saseo
           end
           config
         end
+
+        def reconnect!
+          ActiveRecord::Base.clear_all_connections!
+          ActiveRecord::Base.establish_connection
+        end
       end
 
+      validates :transaction_id, presence: true
+      validates :table_name, presence: true
+      validates :action_timestamp, presence: true
+      validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
+      validates :whodunnit, presence: true
+      validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
+      validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
+
+      validate :validate_has_changes
+
       establish_connection database_config
+
+      def changes
+        old_keys = old_data ? old_data.keys : []
+        new_keys = new_data ? new_data.keys : []
+        (old_keys | new_keys).reduce({}) do |changes, key|
+          old_value = old_data ? old_data[key] : nil
+          new_value = new_data ? new_data[key] : nil
+          changes[key] = [old_value, new_value] if old_value != new_value
+          changes
+        end
+      end
+
+      def has_changes?
+        changes.any?
+      end
+
+      private
+      def validate_has_changes
+        errors.add(:new_data, 'can not be the same as old_data') unless has_changes?
+      end
     end
   end
 end

data/lib/saseo/models/source/version.rb

@@ -6,14 +6,6 @@ module Saseo
     module Source
       class Version < Saseo::Models::Source::Base
         self.table_name = "#{Saseo.config.source_table_schema}.#{Saseo.config.source_table_name}"
-
-        validates :transaction_id, presence: true
-        validates :table_name, presence: true
-        validates :action_timestamp, presence: true
-        validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
-        validates :whodunnit, presence: true
-        validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
-        validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
       end
     end
   end

data/lib/saseo/models/version.rb

@@ -5,14 +5,6 @@ module Saseo
   module Models
     class Version < Saseo::Models::Base
       self.table_name = Saseo.config.table_name
-
-      validates :transaction_id, presence: true
-      validates :table_name, presence: true
-      validates :action_timestamp, presence: true
-      validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
-      validates :whodunnit, presence: true
-      validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
-      validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
     end
   end
 end

data/lib/saseo/persistence/consumer.rb

@@ -5,7 +5,6 @@ require 'saseo/persistence/persistor'
 module Saseo
   module Persistence
     class Consumer < Philotic::Consumer
-      # subscribe_to philotic_message_type: :'saseo.record_audit'
       subscribe_to Saseo.config.consumer_philotic_subscription
 
       manually_acknowledge
@@ -13,7 +12,7 @@ module Saseo
       def consume(message)
         begin
           logger.debug { "received source version: #{message.id}" }
-          Saseo::Persistence::Persistor.persist! message
+          Saseo::Persistence::Persistor.process message
 
           acknowledge message
         rescue => e

data/lib/saseo/persistence/message_filter.rb

@@ -0,0 +1,54 @@
+require 'oj'
+require 'saseo/models/version'
+require 'active_support/core_ext/hash/deep_merge'
+
+module Saseo
+  module Persistence
+    module MessageFilter
+      extend self
+
+      WILD_CARD = '*'
+
+      def filter_message!(message)
+        return message unless ignored_fields_config && filter_match?(ignored_fields_config.keys, message.table_name)
+
+        table_config(message.table_name).each_pair do |fields, ignored_actions|
+          if filter_match?(ignored_actions, message.action)
+            fields_to_remove(fields, message).each do |field|
+              remove_field!(field, message)
+            end
+          end
+        end
+
+        return nil if message.old_data.blank? && message.new_data.blank?
+        message
+      end
+
+      def remove_field!(field, message)
+        message.old_data && message.old_data.delete(field)
+        message.new_data && message.new_data.delete(field)
+      end
+
+      def fields_to_remove(fields, message)
+        fields = Array(fields).map { |f| f.to_s.downcase }
+        fields = message.old_data.keys | message.new_data.keys if fields.include?(WILD_CARD)
+        fields
+      end
+
+      def ignored_fields_config
+        unless defined?(@ignored_fields_config)
+          @ignored_fields_config = Saseo.config.ignore_fields_config_path ? YAML::load(File.open(File.expand_path Saseo.config.ignore_fields_config_path))[Saseo.config.env] : nil
+        end
+        @ignored_fields_config
+      end
+
+      def filter_match?(filters, value)
+        (Array(filters).map { |f| f.to_s.downcase } & [WILD_CARD, value.to_s.downcase]).any?
+      end
+
+      def table_config(table_name)
+        (ignored_fields_config && ignored_fields_config[WILD_CARD] || {}).deep_merge(ignored_fields_config && ignored_fields_config[table_name] || {})
+      end
+    end
+  end
+end

data/lib/saseo/persistence/persistor.rb

@@ -1,5 +1,6 @@
 require 'oj'
 require 'saseo/models/version'
+require 'saseo/persistence/message_filter'
 
 module Saseo
   module Persistence
@@ -25,30 +26,47 @@ module Saseo
         @logger ||= Logger.new(STDOUT)
       end
 
-      def persist!(message)
+      def process(message)
         logger.debug { "received source version for persistence: #{message.id}" }
-        version = Saseo::Models::Version.new
 
-        version_attributes.each do |attr|
-          version.send("#{attr}=", message.send(attr))
+        message = Saseo::Persistence::MessageFilter.filter_message!(message)
+        persist_message!(message) if message
+      end
+
+      def persist_message!(message)
+        save_version!(build_version(message))
+      end
+
+      def build_version(message)
+        Saseo::Models::Version.new.tap do |version|
+
+          version_attributes.each do |attr|
+            version.send("#{attr}=", message.send(attr))
+          end
+
+          # ActiveRecord 3 doesn't handle jsonb columns properly
+          version.old_data = message.old_data && Oj.dump(message.old_data)
+          version.new_data = message.new_data && Oj.dump(message.new_data)
         end
+      end
+
+      def save_version!(version)
+        if version.has_changes?
+          begin
+            version.save!
+            logger.debug { "saved version: #{version.id}" }
 
-        # ActiveRecord 3 doesn't handle jsonb columns properly
-        version.old_data = message.old_data && Oj.dump(message.old_data)
-        version.new_data = message.new_data && Oj.dump(message.new_data)
-
-        begin
-          version.save!
-          logger.debug { "saved version: #{message.id}" }
-        rescue ActiveRecord::RecordNotUnique => e
-          Philotic::Message.publish({philotic_message_type: :'saseo.error.duplicate_record'}, version.attributes)
-        rescue ActiveRecord::StatementInvalid => e
-          if e.message.match /PG::ConnectionBad/
-            logger.warn { 'Lost connection to DB, attempting to reconnect.' }
-            ActiveRecord::Base.clear_all_connections!
-            ActiveRecord::Base.establish_connection
-            retry
+          rescue ActiveRecord::RecordNotUnique => e
+            logger.warn "Duplicate version: #{version.attributes}"
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message.match /PG::ConnectionBad/
+              logger.warn { 'Lost connection to DB, attempting to reconnect.' }
+              Saseo::Models::Version.reconnect!
+              retry
+            end
           end
+        else
+          logger.info { "Skipped version with no changes: #{version.attributes}" }
         end
       end
     end

data/lib/saseo/version.rb

@@ -1,3 +1,3 @@
 module Saseo
-  VERSION = "0.4.3"
+  VERSION = "0.5.0"
 end

data/saseo.gemspec

@@ -25,6 +25,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'generator_spec'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'shoulda-matchers', '~> 2.5'
+  spec.add_development_dependency 'simplecov', '= 0.10'
 
   spec.add_dependency 'oj'
   spec.add_dependency 'pg'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: saseo
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Nathan Keyes
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -108,6 +108,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda-matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -227,12 +255,14 @@ files:
 - lib/saseo/models/version.rb
 - lib/saseo/persistence.rb
 - lib/saseo/persistence/consumer.rb
+- lib/saseo/persistence/message_filter.rb
 - lib/saseo/persistence/persistor.rb
 - lib/saseo/publishing.rb
 - lib/saseo/publishing/data_change_message.rb
 - lib/saseo/publishing/publisher.rb
 - lib/saseo/version.rb
 - lib/saseo/whodunnit.rb
+- sample_saseo_setup.png
 - saseo.example.yml
 - saseo.gemspec
 - tasks/bump.rake