data_cleansing 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d330ec38e5e452e5a6278a9086d2fbd010240d12
-  data.tar.gz: 38dcd89391ba829b8d5b0d31fbf8f5b9c12ac6d0
+  metadata.gz: 963a448b6c2ab7dfe313dd911f19e83a4c4688a6
+  data.tar.gz: ff312d2b3a5dfc3bc20255f909bf3376ef720188
 SHA512:
-  metadata.gz: d7a6e0ff6067d55da2910f494a7f1f3b0688460e794daa9cf7b5e0fdd8e1233df0251573404af4178b4e6d1559cb61960a252a8fad4155a4d773e41090f34867
-  data.tar.gz: 29eb21b9b89ea762e405d80aefe947dc3c2066e563f3f07f2e491bee757cec5cdba5fa239741be8256ba701405278143c3811f61dd973503a0c7470bb547c0e5
+  metadata.gz: 8f899d278990a64a9ce9dcf51713b9143ba9485449ed793f4c96afe9a078dd0273308dfc2eaca41ce081c31ed2beba54173c0b6733c25ef9dfa16b4608e73b9a
+  data.tar.gz: c3d1addfece09468dc7f05981145e98f90ff46721c1b5e68e8b8b283f9b0d71485f0d63164115149b3a2dc84acece390fbbd5fa982b87c8689ad50316f2181bd

data/README.md

@@ -22,11 +22,12 @@ or pull request.
 ## Features
 
 * Supports global cleansing definitions that can be associated with any Ruby,
-  Rails, Mongoid, or other model.
-* Supports custom cleansing definitions that can be defined in-line using block.
-* A cleansing block can access the other attributes in the model in determining
-  how to cleanse the current attribute
-* In a cleansing block other can also be modified if necessary
+  Rails, Mongoid, or other model
+* Supports custom cleansing definitions that can be defined in-line
+* A cleansing block can access the other attributes in the model while cleansing
+  the current attribute
+* In a cleansing block other attributes in the model can be modified at the
+  same time
 * Cleansers are executed in the order they are defined. As a result multiple
   cleansers can be run against the same field and the order is preserved
 * Multiple cleansers can be specified for a list of attributes at the same time
@@ -34,9 +35,8 @@ or pull request.
   the child's cleansers
 * Cleansers can be called outside of a model instance for cases where fields
   need to be cleansed before the model is created, or needs to be found
-* Logging of data cleansing with the before and after values for troubleshooting.
-  Depending on the log level all modified fields are logged, or just the ones
-  completely wiped out to nil
+* To aid troubleshooting the before and after values of cleansed attributes
+  is logged. The level of detail is fine-tuned using the log level
 
 ## ActiveRecord (ActiveModel) Features
 
@@ -67,11 +67,11 @@ u = User.new
 u.first_name = '    joe   '
 u.last_name = "\n  black\n"
 puts "Before data cleansing #{u.inspect}"
-# Before data cleansing Before data cleansing #<User:0x007fc9f1081980 @first_name="    joe   ", @last_name="\n  black\n">
+# Before data cleansing #<User:0x007fc9f1081980 @first_name="    joe   ", @last_name="\n  black\n">
 
 u.cleanse_attributes!
 puts "After data cleansing #{u.inspect}"
-# After data cleansing After data cleansing #<User:0x007fc9f1081980 @first_name="joe", @last_name="black">
+# After data cleansing #<User:0x007fc9f1081980 @first_name="joe", @last_name="black">
 ```
 
 ### Rails Example
@@ -154,6 +154,64 @@ puts "After data cleansing #{u.inspect}"
 # After data cleansing #<User:0x007fdd5a83a8f8 @first_name="joe", @last_name="black", @address1="2632 Brown St", @title="MR.", @gender="Male">
 ```
 
+## After Cleansing
+
+It is sometimes useful to read or write multiple fields as part of a cleansing, or
+where attributes need to be manipulated automatically once they have been cleansed.
+For this purpose instance methods on the model can be registered for invocation once
+all the attributes have been cleansed according to their :cleanse specifications.
+Multiple methods can be registered and they are called in the order they are registered.
+
+```ruby
+after_cleanse <instance_method_name>, <instance_method_name>, ...
+```
+
+Example:
+```ruby
+# Define a global cleanser
+DataCleansing.register_cleaner(:strip) {|string| string.strip}
+
+# 'users' table has the following columns :first_name, :last_name, :address1, :address2
+class User < ActiveRecord::Base
+  include DataCleansing::Cleanse
+
+  # Use a global cleaner
+  cleanse :first_name, :last_name, :cleaner => :strip
+
+  # Define a once off cleaner
+  cleanse :address1, :address2, :cleaner => Proc.new {|string| string.strip}
+
+  # Once the above cleansing is complete call the instance method
+  after_cleanse :check_address
+
+  protected
+
+  # Method to be called once data cleansing is complete
+  def check_address
+    # Move address2 to address1 if Address1 is blank and address2 has a value
+    address2 = address1 if address1.blank? && !address2.blank?
+  end
+
+end
+
+# Create a User instance
+u = User.new(:first_name => '    joe   ', :last_name => "\n  black\n", :address2 => "2632 Brown St   \n")
+puts "Before data cleansing #{u.attributes.inspect}"
+u.cleanse_attributes!
+puts "After data cleansing #{u.attributes.inspect}"
+u.save!
+```
+
+## Recommendations
+
+:data_cleanse block are ideal for cleansing a single attribute, and applying any
+global or common cleansing algorithms.
+
+Even though multiple attributes can be read or written in a single :data_cleanse
+block, it is recommended to use the :after_cleanse method for working with multiple
+attributes. It is much easier to read and understand the interactions between multiple
+attributes in the :after_cleanse methods.
+
 ## Rails configuration
 
 When DataCleansing is used in a Rails environment it can be configured using the
@@ -196,13 +254,22 @@ SemanticLogger.default_level = Rails.logger.level
 SemanticLogger.add_appender(Rails.logger)
 ```
 
-By changing the log level for DataCleansing the type of output for data
+By changing the log level of DataCleansing itself the type of output for data
 cleansing can be controlled:
 
 * :trace or :debug to log all fields modified
 * :info to log only those fields which were nilled out
 * :warn or higher to disable logging of cleansing actions
 
+Note:
+
+* The logging of changes made to attributes only includes attributes cleansed
+  with :data_cleanse blocks. Attributes modified within :after_cleanse methods
+  are not logged
+
+* It is not necessary to change the global log level to affect the logging detail
+  level in DataCleansing. DataCleansing log level is changed independently
+
 To change the log level, either use the Rails configuration approach, or set it
 directly:
 
@@ -212,9 +279,9 @@ DataCleansing.logger.level = :info
 
 ## Notes
 
-Cleaners are called in the order in which they are defined, so subsequent cleaners
-can assume that the previous cleaners have run and can therefore access or even
-modify previously cleaned attributes
+* Cleaners are called in the order in which they are defined, so subsequent cleaners
+  can assume that the previous cleaners have run and can therefore access or even
+  modify previously cleaned attributes
 
 ## Installation
 
@@ -223,7 +290,7 @@ modify previously cleaned attributes
 Add the following line to Gemfile
 
 ```ruby
-gem 'data_validation'
+gem 'data_cleansing'
 ```
 
 Install the Gem with bundler

data/lib/data_cleansing/cleanse.rb

@@ -120,109 +120,122 @@ module DataCleansing
     module InstanceMethods
       # Cleanse the attributes using specified cleaners
       # and execute after cleaners once complete
-      def cleanse_attributes!
-        # Collect parent cleaners first, starting with the top parent
-        cleaners = [self.class.send(:data_cleansing_cleaners)]
-        after_cleaners = [self.class.send(:data_cleansing_after_cleaners)]
-        klass = self.class.superclass
-        while klass != Object
-          cleaners << klass.send(:data_cleansing_cleaners) if klass.respond_to?(:data_cleansing_cleaners)
-          after_cleaners << klass.send(:data_cleansing_after_cleaners) if klass.respond_to?(:data_cleansing_after_cleaners)
-          klass = klass.superclass
-        end
-        cleaners.reverse_each {|cleaner| data_cleansing_execute_cleaners(cleaner)}
+      #
+      # Returns fields changed whilst cleaning the attributes
+      #
+      # Note: At this time the changes returned does not include any fields
+      #       modified in any of the after_cleaner methods
+      def cleanse_attributes!(verbose=DataCleansing.logger.debug?)
+        changes = {}
+        DataCleansing.logger.benchmark_info("#{self.class.name}#cleanse_attributes!", :payload => changes) do
+          # Collect parent cleaners first, starting with the top parent
+          cleaners = [self.class.send(:data_cleansing_cleaners)]
+          after_cleaners = [self.class.send(:data_cleansing_after_cleaners)]
+          klass = self.class.superclass
+          while klass != Object
+            cleaners << klass.send(:data_cleansing_cleaners) if klass.respond_to?(:data_cleansing_cleaners)
+            after_cleaners << klass.send(:data_cleansing_after_cleaners) if klass.respond_to?(:data_cleansing_after_cleaners)
+            klass = klass.superclass
+          end
+          # Capture all modified fields if log_level is :debug or :trace
+          cleaners.reverse_each {|cleaner| changes.merge!(data_cleansing_execute_cleaners(cleaner, verbose))}
 
-        # Execute the after cleaners, starting with the parent after cleanse methods
-        after_cleaners.reverse_each {|a| a.each {|method| send(method)} }
-        true
+          # Execute the after cleaners, starting with the parent after cleanse methods
+          after_cleaners.reverse_each {|a| a.each {|method| send(method)} }
+        end
+        changes
       end
 
       private
 
       # Run each of the cleaners in the order they are listed in the array
-      def data_cleansing_execute_cleaners(cleaners)
+      # Returns a hash of before and after values of what was cleansed
+      # Parameters
+      #   cleaners
+      #     List of cleaners to run
+      #
+      #   verbose [true|false]
+      #     Whether to include all the fields cleansed or just the fields that
+      #     were cleansed to nil
+      def data_cleansing_execute_cleaners(cleaners, verbose = false)
         return false if cleaners.nil?
 
         # Capture all changes to attributes if the log level is :info or greater
-        changes = {} if DataCleansing.logger.info?
-        # Capture all modified fields if log_level is :debug or :trace
-        verbose = DataCleansing.logger.debug?
-
-        DataCleansing.logger.benchmark_info("cleanse_attributes!", :payload => changes) do
-          cleaners.each do |cleaner_struct|
-            params = cleaner_struct.params
-            attrs  = cleaner_struct.attributes
-
-            # Special case to include :all fields
-            # Only works with ActiveRecord based models, not supported with regular Ruby models
-            if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
-              attrs = attributes.keys.collect{|i| i.to_sym}
-              attrs.delete(:id)
-
-              # Remove serialized_attributes if any, from the :all condition
-              if self.class.respond_to?(:serialized_attributes)
-                serialized_attrs = self.class.serialized_attributes.keys
-                attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
-              end
+        changes = {}
+
+        cleaners.each do |cleaner_struct|
+          params = cleaner_struct.params
+          attrs  = cleaner_struct.attributes
+
+          # Special case to include :all fields
+          # Only works with ActiveRecord based models, not supported with regular Ruby models
+          if attrs.include?(:all) && defined?(ActiveRecord) && respond_to?(:attributes)
+            attrs = attributes.keys.collect{|i| i.to_sym}
+            attrs.delete(:id)
+
+            # Remove serialized_attributes if any, from the :all condition
+            if self.class.respond_to?(:serialized_attributes)
+              serialized_attrs = self.class.serialized_attributes.keys
+              attrs -= serialized_attrs.collect{|i| i.to_sym} if serialized_attrs
+            end
 
-              # Replace any encrypted attributes with their non-encrypted versions if any
-              if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
-                self.class.encrypted_attributes.each_pair do |clear, encrypted|
-                  if attrs.include?(encrypted.to_sym)
-                    attrs.delete(encrypted.to_sym)
-                    attrs << clear.to_sym
-                  end
+            # Replace any encrypted attributes with their non-encrypted versions if any
+            if defined?(SymmetricEncryption) && self.class.respond_to?(:encrypted_attributes)
+              self.class.encrypted_attributes.each_pair do |clear, encrypted|
+                if attrs.include?(encrypted.to_sym)
+                  attrs.delete(encrypted.to_sym)
+                  attrs << clear.to_sym
                 end
               end
+            end
 
-              # Explicitly remove specified attributes from cleansing
-              if except = params[:except]
-                attrs -= except
-              end
-
+            # Explicitly remove specified attributes from cleansing
+            if except = params[:except]
+              attrs -= except
             end
 
-            attrs.each do |attr|
-              # Under ActiveModel for Rails and Mongoid need to retrieve raw value
-              # before data type conversion
-              value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
-                read_attribute_before_type_cast(attr.to_s)
-              else
-                send(attr.to_sym)
-              end
+          end
+
+          attrs.each do |attr|
+            # Under ActiveModel for Rails and Mongoid need to retrieve raw value
+            # before data type conversion
+            value = if respond_to?(:read_attribute_before_type_cast) && has_attribute?(attr.to_s)
+              read_attribute_before_type_cast(attr.to_s)
+            else
+              send(attr.to_sym)
+            end
 
-              # No need to clean if attribute is nil
-              unless value.nil?
-                new_value = self.class.send(:data_cleansing_clean,cleaner_struct, value, self)
-
-                if new_value != value
-                  # Update value only if it has changed
-                  send("#{attr.to_sym}=".to_sym, new_value)
-
-                  # Capture changed attributes
-                  if changes
-                    # Mask sensitive attributes when logging
-                    masked = DataCleansing.masked_attributes.include?(attr.to_sym)
-                    new_value = :masked if masked && !new_value.nil?
-                    if previous = changes[attr.to_sym]
-                      previous[:after] = new_value
-                    else
-                      if new_value.nil? || verbose
-                        changes[attr.to_sym] = {
-                          :before => masked ? :masked : value,
-                          :after  => new_value
-                        }
-                      end
+            # No need to clean if attribute is nil
+            unless value.nil?
+              new_value = self.class.send(:data_cleansing_clean,cleaner_struct, value, self)
+
+              if new_value != value
+                # Update value only if it has changed
+                send("#{attr.to_sym}=".to_sym, new_value)
+
+                # Capture changed attributes
+                if changes
+                  # Mask sensitive attributes when logging
+                  masked = DataCleansing.masked_attributes.include?(attr.to_sym)
+                  new_value = :masked if masked && !new_value.nil?
+                  if previous = changes[attr.to_sym]
+                    previous[:after] = new_value
+                  else
+                    if new_value.nil? || verbose
+                      changes[attr.to_sym] = {
+                        :before => masked ? :masked : value,
+                        :after  => new_value
+                      }
                     end
                   end
                 end
               end
-
             end
           end
         end
-        nil
+        changes
       end
+
     end
 
     def self.included(base)

data/lib/data_cleansing/version.rb

@@ -1,3 +1,3 @@
 module DataCleansing
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_cleansing
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-30 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thread_safe