data_cleansing 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8666401eb50e30f3b447a19d1764b6eb6ce34b70
-  data.tar.gz: 4f039f29b178dd10071cb9b8ea97767e0a950250
+  metadata.gz: 2dfd8030076d210f31f5993ed3646abbe38934ea
+  data.tar.gz: a077de74aec84cb805e4087e4e6d04c5c8972a73
 SHA512:
-  metadata.gz: f4949339d972ad1e361363dcf49b6cd6467eb2202c75e8751783818bbda85a31397d582fd9c6eaf72ccb29b379dd515e2dff8c2e07fe3a998d182cf7add187f6
-  data.tar.gz: 9359a1fad55a8f47626c03925a8a956b3b5159624cb1fd6d987b7b3d2a6999c036d9d2677188a5f75ea8551d4d4094a17c351f8f0f1f6f2cf044115fab554008
+  metadata.gz: 9ee068df2896ee7ae1656fc8af6a4baf446b5969972dae873e62833689a3a110e546135a0cd8b37c23847374839aec79b9e12ef8d9c3baf138a1aaf85c00d47d
+  data.tar.gz: dac70b47e27ac5fbf4d96ae12570c071ebb694b69f0f039a798cf6d2a5e28a850845fd03879751eb2fb0eb1b6a11515f8163ee240763d5ac32d4f30d7d7797a2

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
+gem 'thread_safe'
+gem 'semantic_logger'
 
 group :test do
   gem "shoulda"

data/Gemfile.lock

@@ -33,6 +33,9 @@ GEM
       leshill-will_paginate (>= 2.3.11)
       mongo (>= 0.18.2)
     multi_json (1.7.7)
+    semantic_logger (2.1.0)
+      sync_attr (>= 1.0)
+      thread_safe (>= 0.1.0)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
       shoulda-matchers (>= 1.4.1, < 3.0)
@@ -40,6 +43,7 @@ GEM
     shoulda-matchers (2.2.0)
       activesupport (>= 3.0.0)
     sqlite3 (1.3.7)
+    sync_attr (1.0.0)
     thread_safe (0.1.0)
       atomic
     tzinfo (0.3.37)
@@ -53,5 +57,7 @@ DEPENDENCIES
   awesome_print
   jdbc-sqlite3
   mongoid
+  semantic_logger
   shoulda
   sqlite3
+  thread_safe

data/Rakefile

@@ -24,6 +24,7 @@ task :gem  do |t|
     s.license     = "Apache License V2.0"
     s.has_rdoc    = true
     s.add_dependency 'thread_safe'
+    s.add_dependency 'semantic_logger'
   end
   Gem::Package.build gemspec
 end

data/lib/data_cleansing.rb

@@ -1,4 +1,5 @@
 require 'thread_safe'
+require 'semantic_logger'
 require 'data_cleansing/version'
 require 'data_cleansing/data_cleansing'
 
@@ -7,11 +8,6 @@ module DataCleansing
 end
 
 # Rails Extensions
-#if defined?(Rails)
-#  require 'data_cleansing/railtie'
-#end
-
-# Mongoid Extensions
-#if defined?(Mongoid)
-#  require 'data_cleansing/extensions/mongoid/fields'
-#end
+if defined?(Rails)
+  require 'data_cleansing/railtie'
+end

data/lib/data_cleansing/cleanse.rb

@@ -1,102 +1,192 @@
 module DataCleansing
   # Mix-in to add cleaner
   module Cleanse
-    CleanerStruct = Struct.new(:cleaners, :attributes, :params)
+    DataCleansingCleaner = Struct.new(:cleaners, :attributes, :params)
 
     module ClassMethods
       # Define how to cleanse one or more attributes
       def cleanse(*args)
         last = args.last
-        params = (last.is_a?(Hash) && last.instance_of?(Hash)) ? args.pop.dup : {}
+        attributes = args.dup
+        params = (last.is_a?(Hash) && last.instance_of?(Hash)) ? attributes.pop.dup : {}
         cleaners = Array(params.delete(:cleaner))
         raise(ArgumentError, "Mandatory :cleaner parameter is missing: #{params.inspect}") unless cleaners
-        (@cleaners ||= ThreadSafe::Array.new) << CleanerStruct.new(cleaners, args, params)
-        nil
+        cleaner = DataCleansingCleaner.new(cleaners, attributes, params)
+        data_cleansing_cleaners << cleaner
+        attributes.each do |attr|
+          (data_cleansing_attribute_cleaners[attr] ||= ThreadSafe::Array.new) << cleaner
+        end
+        cleaner
+      end
+
+      # Returns the value cleansed using the cleaners defined for that attribute
+      # in this model and any of it's parents
+      #
+      # Parameters
+      #   attribute_name
+      #     Name of the attribute within this Class to be cleansed
+      #   value
+      #     Value to be cleansed
+      #   object
+      #     If supplied the cleansing will be performed within the scope of
+      #     that object so that cleaners can read and write to attributes
+      #     of that object
+      #
+      # Warning: If any of the cleaners read or write to other object attributes
+      #          then a valid object instance must be supplied
+      def cleanse_attribute(attribute_name, value, object=nil)
+        return if value.nil?
+
+        # Collect parent cleaners first, starting with the top parent
+        cleaners = []
+        klass = self
+        while klass != Object
+          if klass.respond_to?(:data_cleansing_attribute_cleaners)
+            cleaners += klass.data_cleansing_attribute_cleaners[:all] || []
+            cleaners += klass.data_cleansing_attribute_cleaners[attribute_name.to_sym] || []
+          end
+          klass = klass.superclass
+        end
+        cleansed_value = value.dup
+        cleaners.reverse_each {|cleaner| cleansed_value = data_cleansing_clean(cleaner, cleansed_value, object) if cleaner}
+        cleansed_value
       end
 
-      def cleaners
-        @cleaners
+      # Array of cleaners to execute against this model and it's children
+      def data_cleansing_cleaners
+        @data_cleansing_cleaners ||= ThreadSafe::Array.new
+      end
+
+      # Hash of attributes to clean with their corresponding cleaner
+      def data_cleansing_attribute_cleaners
+        @data_cleansing_attribute_cleaners ||= ThreadSafe::Hash.new
+      end
+
+      private
+
+      # Returns the supplied value cleansed using the supplied cleaner
+      # Parameters
+      #   object
+      #     If supplied the cleansing will be performed within the scope of
+      #     that object so that cleaners can read and write to attributes
+      #     of that object
+      #
+      # No logging of cleansing is performed by this method since the value
+      # itself is not modified
+      def data_cleansing_clean(cleaner_struct, value, object=nil)
+        return if cleaner_struct.nil? || value.nil?
+        # Duplicate value in case cleaner uses methods such as gsub!
+        new_value = value.is_a?(String) ? value.dup : value
+        cleaner_struct.cleaners.each do |cleaner|
+          # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
+          proc = cleaner.is_a?(Proc) ? cleaner : DataCleansing.cleaner(cleaner.to_sym)
+          raise "No cleaner defined for #{cleaner.inspect}" unless proc
+
+          new_value = if object
+            # Call the cleaner proc within the scope (binding) of the object
+            proc.arity == 1 ? object.instance_exec(new_value, &proc) : object.instance_exec(new_value, cleaner_struct.params, &proc)
+          else
+            proc.arity == 1 ? proc.call(new_value) : proc.call(new_value, cleaner_struct.params)
+          end
+        end
+        new_value
       end
+
     end
 
     module InstanceMethods
       # Cleanse the attributes using specified cleaners
       def cleanse_attributes!
-        # Execute parent cleansers first, starting with the top parent
-        klasses = [self.class]
+        # Collect parent cleaners first, starting with the top parent
+        cleaners = [self.class.send(:data_cleansing_cleaners)]
         klass = self.class.superclass
         while klass != Object
-          klasses << klass if klass.respond_to?(:cleaners)
+          cleaners << klass.send(:data_cleansing_cleaners) if klass.respond_to?(:data_cleansing_cleaners)
           klass = klass.superclass
         end
-        klasses.reverse_each {|klass| execute_cleaners(klass.cleaners)}
+        cleaners.reverse_each {|cleaner| data_cleansing_execute_cleaners(cleaner)}
         true
       end
 
       private
 
       # Run each of the cleaners in the order they are listed in the array
-      def execute_cleaners(cleaners)
+      def data_cleansing_execute_cleaners(cleaners)
         return false if cleaners.nil?
-        cleaners.each do |cleaner_struct|
-          params = cleaner_struct.params
-          attrs  = cleaner_struct.attributes
-
-          # Special case to include :all fields
-          # Only works with ActiveRecord based models, not supported with regular Ruby models
-          if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
-            attrs = attributes.keys.collect{|i| i.to_sym}
-            attrs.delete(:id)
-
-            # Remove serialized_attributes if any, from the :all condition
-            if self.class.respond_to?(:serialized_attributes)
-              serialized_attrs = self.class.serialized_attributes.keys
-              attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
-            end
 
-            # Replace any encrypted attributes with their non-encrypted versions if any
-            if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
-              self.class.encrypted_attributes.each_pair do |clear, encrypted|
-                if attrs.include?(encrypted.to_sym)
-                  attrs.delete(encrypted.to_sym)
-                  attrs << clear.to_sym
-                end
+        # Capture all changes to attributes if the log level is high enough
+        changes = {} if DataCleansing.logger.send("#{DataCleansing.cleansing_log_level}?")
+
+        #logger.send(self.class.data_cleansing_log_level, "Cleansed Attributes", changes) if changes
+        DataCleansing.logger.send("benchmark_#{DataCleansing.cleansing_log_level}","Cleansed Attributes", :payload => changes) do
+          cleaners.each do |cleaner_struct|
+            params = cleaner_struct.params
+            attrs  = cleaner_struct.attributes
+
+            # Special case to include :all fields
+            # Only works with ActiveRecord based models, not supported with regular Ruby models
+            if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
+              attrs = attributes.keys.collect{|i| i.to_sym}
+              attrs.delete(:id)
+
+              # Remove serialized_attributes if any, from the :all condition
+              if self.class.respond_to?(:serialized_attributes)
+                serialized_attrs = self.class.serialized_attributes.keys
+                attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
               end
-            end
 
-            # Explicitly remove specified attributes from cleansing
-            if except = params[:except]
-              attrs -= except
-            end
+              # Replace any encrypted attributes with their non-encrypted versions if any
+              if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
+                self.class.encrypted_attributes.each_pair do |clear, encrypted|
+                  if attrs.include?(encrypted.to_sym)
+                    attrs.delete(encrypted.to_sym)
+                    attrs << clear.to_sym
+                  end
+                end
+              end
 
-          end
+              # Explicitly remove specified attributes from cleansing
+              if except = params[:except]
+                attrs -= except
+              end
 
-          attrs.each do |attr|
-            # Under ActiveModel for Rails and Mongoid need to retrieve raw value
-            # before data type conversion
-            value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
-              read_attribute_before_type_cast(attr.to_s)
-            else
-              send(attr.to_sym)
             end
 
-            # No need to clean if attribute is nil
-            unless value.nil?
-              # Allow multiple cleaners to be defined and only set the new value
-              # once all cleaners have run
-              new_value = value
-              cleaner_struct.cleaners.each do |cleaner|
-                # Cleaner itself could be a custom Proc, otherwise do a global lookup for it
-                proc = cleaner.is_a?(Proc) ? cleaner : DataCleansing.cleaner(cleaner.to_sym)
-                raise "No cleaner defined for #{cleaner.inspect}" unless proc
-
-                # Call the cleaner proc within the scope (binding) of this object
-                new_value = instance_exec(new_value, params, &proc)
+            attrs.each do |attr|
+              # Under ActiveModel for Rails and Mongoid need to retrieve raw value
+              # before data type conversion
+              value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
+                read_attribute_before_type_cast(attr.to_s)
+              else
+                send(attr.to_sym)
               end
-              # Update value if it has changed
-              send("#{attr.to_sym}=".to_sym, new_value) if new_value != value
-            end
 
+              # No need to clean if attribute is nil
+              unless value.nil?
+                new_value = self.class.send(:data_cleansing_clean,cleaner_struct, value, self)
+
+                if new_value != value
+                  # Update value only if it has changed
+                  send("#{attr.to_sym}=".to_sym, new_value)
+
+                  # Capture changed attributes
+                  if changes
+                    # Mask sensitive attributes when logging
+                    masked = DataCleansing.masked_attributes.include?(attr.to_sym)
+                    new_value = :masked if masked && !new_value.nil?
+                    if previous = changes[attr.to_sym]
+                      previous[:after] = new_value
+                    else
+                      changes[attr.to_sym] = {
+                        :before => masked ? :masked : value,
+                        :after  => new_value
+                      }
+                    end
+                  end
+                end
+              end
+
+            end
           end
         end
         nil
@@ -105,8 +195,8 @@ module DataCleansing
 
     def self.included(base)
       base.class_eval do
-        extend(DataCleansing::Cleanse::ClassMethods)
-        include(DataCleansing::Cleanse::InstanceMethods)
+        extend  DataCleansing::Cleanse::ClassMethods
+        include DataCleansing::Cleanse::InstanceMethods
       end
     end
   end

data/lib/data_cleansing/data_cleansing.rb

@@ -1,23 +1,41 @@
 module DataCleansing
+  include SemanticLogger::Loggable
 
   # Global Data Cleansers
-  @@global_cleaners = ThreadSafe::Hash.new
+  @@global_cleaners     = ThreadSafe::Hash.new
+  @@masked_attributes   = ThreadSafe::Array.new
+  @@cleansing_log_level = :info
 
   # Register a new cleaner
   # Replaces any existing cleaner with the same name
   def self.register_cleaner(cleaner, &block)
-    if block
-      @@global_cleaners[cleaner.to_sym] = block
-    else
-      # TODO Expose class methods as cleaners
-      #
-      # cleaners[cleaner.to_sym] = block
-      # raise ArgumentError, "Must supply either a Proc, or a cleaner klass"
-    end
+    raise "Must supply a Proc with the cleaner" unless block
+    @@global_cleaners[cleaner.to_sym] = block
   end
 
   # Returns the cleaner matching the supplied cleaner name
   def self.cleaner(cleaner_name)
     @@global_cleaners[cleaner_name.to_sym]
   end
+
+  # Register Attributes to be masked out in any log output
+  def self.register_masked_attributes(*attributes)
+    attributes.each {|attr| @@masked_attributes << attr.to_sym }
+  end
+
+  # Returns the Global list of attributes to mask in any log output
+  def self.masked_attributes
+    @@masked_attributes.freeze
+  end
+
+  # Set the log_level at which to log cleansing activities at
+  def self.cleansing_log_level
+    @@cleansing_log_level
+  end
+
+  # Set the log_level at which to log cleansing activities at
+  def self.cleansing_log_level=(log_level)
+    @@cleansing_log_level = log_level
+  end
+
 end

data/lib/data_cleansing/railtie.rb

@@ -0,0 +1,16 @@
+module RubySkynet #:nodoc:
+  class Railtie < Rails::Railtie #:nodoc:
+
+    # Exposes DataCleansing configuration to the Rails application configuration.
+    #
+    # @example Set up configuration in the Rails app.
+    #   module MyApplication
+    #     class Application < Rails::Application
+    #       config.data_cleansing.masked_attributes :bank_account_number, :social_security_number
+    #       config.data_cleansing.cleansing_log_level :info
+    #       config.data_cleansing.register_cleaner(:strip) {|string| string.strip!}
+    #     end
+    #   end
+    config.data_cleansing = ::DataCleansing
+  end
+end

data/lib/data_cleansing/version.rb

@@ -1,3 +1,3 @@
 module DataCleansing
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/test/active_record_test.rb

@@ -8,8 +8,13 @@ require 'shoulda'
 # are loaded
 require 'active_record'
 require 'data_cleansing'
+require 'semantic_logger'
 
-ActiveRecord::Base.logger = Logger.new($stderr)
+# Register an appender if one is not already registered
+SemanticLogger.default_level = :trace
+SemanticLogger.add_appender('test.log') if SemanticLogger.appenders.size == 0
+
+ActiveRecord::Base.logger = SemanticLogger[ActiveRecord::Base]
 ActiveRecord::Base.configurations = {
   'test' => {
     'adapter'  => 'sqlite3',
@@ -26,6 +31,7 @@ ActiveRecord::Schema.define :version => 0 do
     t.string  :last_name
     t.string  :address1
     t.string  :address2
+    t.string  :ssn
     t.integer :zip_code
   end
 end
@@ -33,6 +39,13 @@ end
 # Define a global cleaner
 DataCleansing.register_cleaner(:strip) {|string, params, object| string.strip!}
 
+# Log data cleansing result at :info level
+# Set to a level below the current log level to disable
+DataCleansing.cleansing_log_level = :info
+
+# Set the Global list of fields to be masked
+DataCleansing.register_masked_attributes :ssn, :bank_account_number
+
 # Removes all non-digit characters, except '.' then truncates
 # the result to an integer string
 # Returns nil if no digits are present in the string
@@ -74,6 +87,15 @@ class User2 < ActiveRecord::Base
   # Test :all cleaner. Only works with ActiveRecord Models
   cleanse :all, :cleaner => [:strip, Proc.new{|s| "@#{s}@"}], :except => [:address1, :zip_code]
 
+  # Clean :first_name multiple times
+  cleanse :first_name, :cleaner => Proc.new {|string| "<< #{string} >>"}
+
+  # Clean :first_name multiple times
+  cleanse :first_name, :cleaner => Proc.new {|string| "$#{string}$"}
+
+  # Custom Zip Code cleaner
+  cleanse :zip_code, :cleaner => :digits_to_integer
+
   # Automatically cleanse data before validation
   before_validation :cleanse_attributes!
 end
@@ -85,19 +107,27 @@ class ActiveRecordTest < Test::Unit::TestCase
       assert DataCleansing.cleaner(:strip)
     end
 
+    should 'Model.cleanse_attribute' do
+      assert_equal 'joe',                 User.cleanse_attribute(:first_name,   '    joe   ')
+      assert_equal 'black',               User.cleanse_attribute(:last_name,    "\n  black\n")
+      assert_equal '<< 2632 Brown St >>', User.cleanse_attribute(:address1,     "2632 Brown St   \n")
+      assert_equal '<< instance >>',      User.cleanse_attribute(:instance_var, "\n    instance\n\t      ")
+      assert_equal 12345,                 User.cleanse_attribute(:zip_code,     "\n\tblah 12345badtext\n")
+    end
+
     context "with user" do
       setup do
         @user = User.new(
-          :first_name => '    joe   ',
-          :last_name  => "\n  black\n",
-          :address1   => "2632 Brown St   \n",
-          :zip_code   => "\n\tblah 12345badtext\n",
+          :first_name   => '    joe   ',
+          :last_name    => "\n  black\n",
+          :address1     => "2632 Brown St   \n",
+          :zip_code     => "\n\tblah 12345badtext\n",
           :instance_var => "\n    instance\n\t      "
         )
       end
 
       should 'only have 3 cleaners' do
-        assert_equal 3, User.cleaners.size, User.cleaners
+        assert_equal 3, User.send(:data_cleansing_cleaners).size, User.send(:data_cleansing_cleaners)
       end
 
       should 'cleanse_attributes! using global cleaner' do
@@ -123,21 +153,28 @@ class ActiveRecordTest < Test::Unit::TestCase
         @user = User2.new(
           :first_name => '    joe   ',
           :last_name  => "\n  black\n",
+          :ssn        => "\n    123456789   \n  ",
           :address1   => "2632 Brown St   \n",
-          :zip_code   => "\n\t12345\n"
+          :zip_code   => "\n\t  blah\n"
         )
       end
 
-      should 'only have 1 cleaner' do
-        assert_equal 1, User2.cleaners.size, User2.cleaners
+      should 'have 4 cleaners defined' do
+        assert_equal 4, User2.send(:data_cleansing_cleaners).size, User2.send(:data_cleansing_cleaners)
+      end
+
+      should 'have 3 attributes cleaners defined' do
+        # :all, :first_name, :zip_code
+        assert_equal 3, User2.send(:data_cleansing_attribute_cleaners).size, User2.send(:data_cleansing_attribute_cleaners)
       end
 
       should 'cleanse_attributes! clean all attributes' do
         assert_equal true, @user.valid?
-        assert_equal '@joe@', @user.first_name, User2.cleaners
+        assert_equal '$<< @joe@ >>$', @user.first_name, User2.send(:data_cleansing_cleaners)
         assert_equal '@black@', @user.last_name
         assert_equal "2632 Brown St   \n", @user.address1
-        assert_equal 12345, @user.zip_code, User2.cleaners
+        assert_equal "@123456789@", @user.ssn
+        assert_equal nil, @user.zip_code, User2.send(:data_cleansing_cleaners)
       end
 
     end

data/test/ruby_test.rb

@@ -70,6 +70,13 @@ class RubyTest < Test::Unit::TestCase
       assert DataCleansing.cleaner(:strip)
     end
 
+    should 'Model.cleanse_attribute' do
+      assert_equal 'male',                RubyUserChild.cleanse_attribute(:gender,     "\n   Male   \n"), RubyUserChild.send(:data_cleansing_attribute_cleaners)
+      assert_equal 'joe',                 RubyUserChild.cleanse_attribute(:first_name, '    joe   '), RubyUserChild.send(:data_cleansing_attribute_cleaners)
+      assert_equal 'black',               RubyUserChild.cleanse_attribute(:last_name,  "\n  black\n"), RubyUserChild.send(:data_cleansing_attribute_cleaners)
+      assert_equal '<< 2632 Brown St >>', RubyUserChild.cleanse_attribute(:address1,   "2632 Brown St   \n"), RubyUserChild.send(:data_cleansing_attribute_cleaners)
+    end
+
     context "with ruby user" do
       setup do
         @user = RubyUser.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_cleansing
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-19 00:00:00.000000000 Z
+date: 2013-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thread_safe
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Data Cleansing framework for Ruby with additional support for Rails and
   Mongoid
 email:
@@ -40,6 +54,7 @@ files:
 - lib/data_cleansing.rb
 - lib/data_cleansing/cleanse.rb
 - lib/data_cleansing/data_cleansing.rb
+- lib/data_cleansing/railtie.rb
 - lib/data_cleansing/version.rb
 - nbproject/private/private.properties
 - nbproject/private/private.xml